This repository has been archived on 2023-06-18. You can view files and clone it, but cannot push or open issues or pull requests.
soft-arch/build.js

688 lines
19 KiB
JavaScript
Raw Normal View History

2023-02-21 19:09:29 +00:00
const { PerformanceObserver, performance } = require('perf_hooks');
let t_start = performance.now();
process.on('exit', function () {
let t_end = performance.now();
console.log("Done in " + (t_end - t_start) + "ms");
});
const fs = require('fs-extra');
const glob = require('glob-fs')({ gitignore: true });
const marked = require('marked');
const marked_toc = require('marked');
const plantuml = require('plantuml');
const _ = require('lodash');
const slugify = require('uslug');
const { transformFml } = require('./fmlTransformer')
2023-02-21 19:09:29 +00:00
const fml_syntax_highlight = require('./fml_syntax_highlight');
2023-02-21 19:09:29 +00:00
const child_process = require('child_process');
function run(cmd, cwd, done) {
console.log(cmd);
const exec = child_process.exec;
exec(cmd, { cwd }, (err, stdout, stderr) => {
if (err) {
//some err occurred
console.error(err);
} else {
// the *entire* stdout and stderr (buffered)
console.log(`stdout: ${stdout}`);
console.log(`stderr: ${stderr}`);
if (typeof done == "function") {
done(err, stdout, stderr);
} else { }
}
});
}
const build_json = fs.readJSONSync('build.json', { throws: false }) || {};
build_json.doc_build = (build_json.doc_build || 0) + 1;
fs.outputJsonSync('build.json', build_json, { spaces: 2 });
String.prototype.hashCode = function () {
var hash = 0, i, chr;
if (this.length === 0) return hash;
for (i = 0; i < this.length; i++) {
chr = this.charCodeAt(i);
hash = ((hash << 5) - hash) + chr;
hash |= 0; // Convert to 32bit integer
}
return hash;
};
const cache_json = fs.readJSONSync('./upload/puml_cache.json', { throws: false }) || {};
function cache_lookup(text, miss) {
let hash = new String(text).hashCode();
if (cache_json[hash]) { //hit
return new Promise((resolve, reject) => resolve(cache_json[hash]));
} else {
return miss(text).then((output) => {
cache_json[hash] = output;
fs.outputJson('./upload/puml_cache.json', cache_json, { spaces: 2 });
return output;
})
}
}
let puml_count = 0;
let epuml_count = 0;
let fml_count = 0;
function toc(str) {
// var defaultTemplate = '<%= depth %><%= bullet %>[<%= heading %>](#<%= url %>)\n';
var defaultTemplate = '<a class="d<%=d%>" href="#<%= url %>"><%- heading %></a>\n';
2023-02-21 19:09:29 +00:00
const utils = {};
utils.arrayify = function (arr) {
return !Array.isArray(arr) ? [arr] : arr;
};
utils.escapeRegex = function (re) {
return re.replace(/(\[|\]|\(|\)|\/|\.|\^|\$|\*|\+|\?)/g, '\\$1');
};
utils.isDest = function (dest) {
return !dest || dest === 'undefined' || typeof dest === 'object';
};
utils.isMatch = function (keys, str) {
keys = utils.arrayify(keys);
keys = (keys.length > 0) ? keys.join('|') : '.*';
// Escape certain characters, like '[', '('
var k = utils.escapeRegex(String(keys));
// Build up the regex to use for replacement patterns
var re = new RegExp('(?:' + k + ')', 'g');
if (String(str).match(re)) {
return true;
} else {
return false;
}
};
utils.sanitize = function (src) {
src = src.replace(/(\s*\[!|(?:\[.+ →\]\()).+/g, '');
src = src.replace(/\s*\*\s*\[\].+/g, '');
return src;
};
utils.slugify = function (str) {
str = str.replace(/\/\//g, '-');
str = str.replace(/\//g, '-');
str = str.replace(/\./g, '-');
str = _.str.slugify(str);
str = str.replace(/^-/, '');
str = str.replace(/-$/, '');
return str;
};
var omit = ['grunt', 'helper', 'handlebars-helper', 'mixin', 'filter', 'assemble-contrib', 'assemble'];
utils.strip = function (name, options) {
var opts = _.extend({}, options);
if (opts.omit === false) { omit = []; }
var exclusions = _.union(omit, utils.arrayify(opts.strip || []));
var re = new RegExp('^(?:' + exclusions.join('|') + ')[-_]?', 'g');
return name.replace(re, '');
2023-02-21 19:09:29 +00:00
};
var opts = {
firsth1: false,
blacklist: true,
omit: [],
maxDepth: 3,
slugifyOptions: { allowedChars: '-' },
slugify: function (text) {
return slugify(text, opts.slugifyOptions).replace("ex-", "ex---");
2023-02-21 19:09:29 +00:00
}
};
var toc = '';
var tokens = marked.lexer(str);
var tocArray = [];
// Remove the very first h1, true by default
if (opts.firsth1 === false) {
tokens.shift();
}
// Do any h1's still exist?
var h1 = _.some(tokens, { depth: 1 });
var task_count = 0;
tokens.filter(function (token) {
// Filter out everything but headings
if (token.type !== 'heading' || token.type === 'code') {
return false;
}
// Since we removed the first h1, we'll check to see if other h1's
// exist. If none exist, then we unindent the rest of the TOC
if (!h1) {
token.depth = token.depth - 1;
}
// Store original text and create an id for linking
token.heading = opts.strip ? utils.strip(token.text, opts) : token.text;
if (token.heading.indexOf("Ex -") >= 0) {
token.heading = token.heading.replace("Ex -", `<span class='task'>${task_count}.</span> `);
2023-03-07 16:09:02 +00:00
task_count++;
2023-05-01 15:05:12 +00:00
if (task_count === 13) task_count++; // skip hands-on
2023-02-21 19:09:29 +00:00
}
// Create a "slugified" id for linking
token.id = opts.slugify(token.text);
// Omit headings with these strings
var omissions = ['Table of Contents', 'TOC', 'TABLE OF CONTENTS'];
var omit = _.union([], opts.omit, omissions);
if (utils.isMatch(omit, token.heading)) {
return;
}
return true;
}).forEach(function (h) {
if (h.depth > opts.maxDepth) {
return;
}
var bullet = Array.isArray(opts.bullet)
? opts.bullet[(h.depth - 1) % opts.bullet.length]
: opts.bullet;
var data = _.extend({}, opts.data, {
d: h.depth,
depth: new Array((h.depth - 1) * 2 + 1).join(' '),
bullet: bullet ? bullet : '* ',
heading: h.heading,
url: h.id
});
tocArray.push(data);
toc += ejs.render(defaultTemplate, data);
2023-02-21 19:09:29 +00:00
//toc += _.template(opts.template || defaultTemplate, data);
});
return {
data: tocArray,
toc: opts.strip
? utils.strip(toc, opts)
: toc
};
}
const renderer = new marked.Renderer();
renderer._paragraph = renderer.paragraph;
renderer.paragraph = function (text) {
//console.log(text);
if (text.trim().startsWith("{")) {
renderer.sec_depth = renderer.sec_depth || 0;
renderer.sec_depth++;
let c = text.split(".");
c.shift();
if (c.length > 0) {
return `<section class="${c.join(" ")}">`;
} else {
return "<section>"
}
}
if (text.trim().startsWith("}")) {
renderer.sec_depth = renderer.sec_depth || 0;
renderer.sec_depth--;
return "</section>"
}
if (text.trim().startsWith("Pass: ") && renderer.sec_depth > 0) {
return `<section class="pass"><span>&starf;&star;&star;: </span>${text.split("Pass: ")[1]}</section>`;
}
if (text.trim().startsWith("Good: ") && renderer.sec_depth > 0) {
return `<section class="ok"><span>&starf;&starf;&star;: </span>${text.split("Good: ")[1]}</section>`;
}
if (text.trim().startsWith("Exceed: ") && renderer.sec_depth > 0) {
return `<section class="exceed"><span>&starf;&starf;&starf;: </span>${text.split("Exceed: ")[1]}</section>`;
}
if (text.trim().startsWith("Hint: ") && renderer.sec_depth > 0) {
return `<section class="hint"><span>Hint: </span>${text.split("Hint: ")[1]}</section>`;
}
return renderer._paragraph(text);
}
renderer._code = renderer.code;
renderer.code = function (code, infostring, escaped) {
// console.log(code, infostring, escaped);
2023-02-21 19:09:29 +00:00
if (infostring) {
if (infostring.startsWith("puml")) {
let i = epuml_count++;
cache_lookup(code, () => plantuml(code)).then(svg => {
fs.writeFile(renderer._out_folder + "puml_e" + i + ".svg", svg)
})
return `<img src="./puml_e${i}.svg">`;
}
if (infostring.startsWith("fml1") || infostring.startsWith("fml2")) {
let fml = "" + code;
let i = fml_count++;
const fml_output = renderer._out_folder + "fml_" + i + ".svg";
const tempFile = renderer._out_folder + `fml_${i}.temp.fml`;
try{
fs.ensureFileSync(tempFile);
const transformedFml = transformFml(code, infostring.startsWith("fml1") ? "Flat" : "Nested")
fs.writeFile(tempFile, transformedFml)
.then(_ => {
run("node fml " + tempFile + " " + fml_output, undefined, () => {
fs.remove(tempFile)
})
})
if (infostring.indexOf("?src") >= 0) {
return `<aside class="flex"><pre><code>${highlight_fml(fml)}</code></pre><img src="./fml_${i}.svg"></aside>`;
} else {
return `<img src="./fml_${i}.svg">`;
}
} catch(e) {
//the original code will be printed (opportunity to show the error)
console.log(e);
}
}
2023-02-21 19:09:29 +00:00
}
return renderer._code(code, infostring, escaped);
};
//TODO (["'])(?:(?=(\\?))\2.)*?\1 for quoted strings (which do not contain keywords)
function highlight_fml(s) {
return fml_syntax_highlight.parse(s);
let keywords = ["feature", "optional", "required", "hasOneOf", "hasSome", "has", "requires", "excludes"] //fml1
return keywords.reduce((s, k) => s.replace(new RegExp(k+" ", "g"), `<b>${k}</b> `, s)
.replace(new RegExp(k+"\n", "g"), `<b>${k}</b>\n`, s)
.replace(new RegExp(k, "g"), `<b>${k}</b>`), s);
}
2023-02-21 19:09:29 +00:00
renderer._image = renderer.image;
renderer.image = function (href, title, text) {
//console.log(href, title, text);
let url = new URL(href, "http://localhost/");
href = "." + url.pathname;
2023-02-21 19:09:29 +00:00
if (href.endsWith(".puml")) {
let i = puml_count++;
fs.readFile(href.replace("./", renderer._in_folder)).then(puml => {
return cache_lookup(puml, () => plantuml(puml))
//return plantuml(puml)
}).then(svg => {
fs.writeFile(renderer._out_folder + "puml_" + i + ".svg", svg)
})
return `<img src="./puml_${i}.svg" alt="${text}">`;
}
if (href.endsWith(".fml1") || href.endsWith(".fml2")) {
let i = fml_count++;
const fml_output = renderer._out_folder + "fml_" + i + ".svg";
const tempFile = renderer._out_folder + `fml_${i}.temp.fml`;
fs.ensureFileSync(tempFile);
let fml = fs.readFileSync(href.replace("./", renderer._in_folder), 'utf8');
const transformedFml = transformFml(fml, href.endsWith(".fml1") ? "Flat" : "Nested")
fs.writeFile(tempFile, transformedFml)
.then(_ => {
run("node fml " + tempFile + " " + fml_output, undefined, () => {
fs.remove(tempFile)
})
})
if (url.searchParams.has("src")) {
return `<aside class="flex"><pre><code>${highlight_fml(fml)}</code></pre><img src="./fml_${i}.svg" alt="${text}"></aside>`;
} else {
return `<img src="./fml_${i}.svg" alt="${text}">`;
}
}
if (href.endsWith(".fml") || href.endsWith(".fml0")) {
2023-02-21 19:09:29 +00:00
let i = fml_count++;
const fml_output = renderer._out_folder + "fml_" + i + ".svg";
run("node fml " + href.replace("./", renderer._in_folder) + " " + fml_output);
return `<img src="./fml_${i}.svg" alt="${text}">`;
}
if (href.endsWith(".c5")) {
let i = fml_count++;
const fml_output = renderer._out_folder + "c5_" + i + ".svg";
run("node c5 " + href.replace("./", renderer._in_folder) + " " + fml_output);
return `<img src="./c5_${i}.svg" alt="${text}">`;
}
if (href.endsWith(".madr") || href.endsWith(".md")) {
2023-02-21 19:09:29 +00:00
let md = fs.readFileSync(href.replace("./", renderer._in_folder));
return `<h3>${text}</h3>` + marked.parse("" + new String(md));
}
if (href.endsWith(".svg") || href.endsWith(".png") ) {
let fileName = href.split("/").slice(-1);
let out = renderer._out_folder + `${fileName}`;
fs.copySync(href.replace("./", renderer._in_folder), out);
return `<img src="${fileName}" alt="${text}">`;
}
2023-02-21 19:09:29 +00:00
return renderer._image(href, title, text);
}
renderer._heading = renderer.heading;
let task_count = 0;
renderer.heading = function (text, level, raw, slugger) {
// console.log(text);
if (text.indexOf("Ex -") >= 0) {
text = text.replace("Ex -", `<span class='task'>${task_count}.</span> `);
2023-03-07 16:09:02 +00:00
task_count++;
2023-05-01 15:05:12 +00:00
if (task_count === 13) task_count++; // skip hands-on
2023-02-21 19:09:29 +00:00
}
// let sec = "<section>"
// if (task_count > 1) {
// sec = "</section>" + sec;
// }
return renderer._heading(text, level, raw, slugger);
};
marked.setOptions({
renderer, //: new marked.Renderer(),
highlight: function (code) {
return code; //require('highlight').highlightAuto(code).value;
},
pedantic: false,
gfm: true,
breaks: false,
sanitize: false,
smartLists: true,
smartypants: false,
xhtml: false
});
function loadMD(f, vars = {}) {
let txt = "" + new String(fs.readFileSync(f));
if (txt.indexOf("---") >= 0) {
let parts = txt.split("\n---\n");
parts.slice(0, -1).forEach(p => {
let lines = p.split("\n");
let k = lines[0].split(":");
vars[k[0]] = lines.slice(1).join("\n");
});
txt = parts[parts.length - 1];
}
//convert strings to booleans
Object.keys(vars).forEach(k => {
if (vars[k] === "false") vars[k] = false;
if (vars[k] === "true") vars[k] = true;
})
console.log(vars);
task_count = 0;
return { md: marked.parse(txt), meta: vars, toc: toc(txt) };
}
let ejs = require('ejs');
let views = {}
function render_one(model, out_file, template) {
model.out_file = out_file;
views[template] = views[template] || new String(fs.readFileSync("./template/" + template + ".ejs"));
let html = ejs.render(views[template], model);
fs.ensureFileSync(out_file);
fs.writeFileSync(out_file, html);
};
var walk = function (dir, done) {
var results = [];
fs.readdir(dir, function (err, list) {
if (err) return done(err);
var i = 0;
(function next() {
var file = list[i++];
if (!file) return done(null, results);
file = dir + '/' + file;
fs.stat(file, function (err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function (err, res) {
results = results.concat(res);
next();
});
} else {
results.push({ name: file, stat });
next();
}
});
})();
});
};
function deltaBuild() {
let old_file_list = {};
try {
old_file_list = fs.readJsonSync('upload/files.json');
} catch (e) { }
walk("src", function (err, list) {
if (list === undefined) {
console.log("Missing src folder")
return;
}
if (list.length == 0) {
console.log("Empty src folder")
return;
}
let flat = {};
list.filter((f) => !f.name.endsWith(".DS_Store")).forEach(f => {
flat[f.name] = f.stat;
});
let todo = {};
let deleted = {};
Object.keys(old_file_list).forEach(k => { deleted[k] = old_file_list[k] });
let changed = [];
function extract_dir(k) {
let a = k.split("/")
return a.slice(1, 3).join("/");
}
function push(k) {
let d = extract_dir(k);
todo[d] = (todo[d] || 0) + 1;
changed.push(k);
}
Object.keys(flat).forEach((k) => {
if (old_file_list[k]) {
if (flat[k].mtimeMs != old_file_list[k].mtimeMs) {
push(k);
}
delete deleted[k]; //k was found so what's left over should have been deleted
} else {
push(k); //this is a new addition
}
});
fs.writeJson("upload/files.json", flat);
console.log('Changed', changed);
console.log('Deleted', Object.keys(deleted));
if (Object.keys(todo).length == 0) {
todo['sa/model'] = 1;
}
//todo['exercises/8-ws'] = 1;
if (Object.keys(todo).indexOf('templates/doc.ejs') != -1) {
todo = Array.from(new Set(Object.keys(flat).map(extract_dir))).reduce((a, c) => { a[c] = 1; return a; }, {});
delete todo['templates/doc.ejs'];
}
Object.keys(todo).filter(k => k.endsWith('meta.json')).forEach(mk => {
delete todo[mk];
Array.from(new Set(Object.keys(flat).map(extract_dir))).filter(k => k.startsWith(mk.split("/")[0])).forEach(k => {
todo[k] = 1;
})
})
console.log("Building", todo);
Object.keys(todo).forEach(buildDoc.bind(null, changed, Object.keys(deleted)));
})
}
function buildDoc(changed, deleted, slide) {
console.log("Building " + slide);
let md_path = "./src/" + slide + "/index.md";
let md = { meta: {} };
let meta_path = "./src/" + slide.split("/")[0] + "/meta.json";
try {
md.meta = fs.readJSONSync(meta_path);
} catch (e) {
//console.log(e);
}
md.meta.build = build_json.doc_build;
//md.meta.lecture = "Web Atelier 2020 ";
md.meta.timestamp = new Date().toLocaleString();
if (fs.existsSync(md_path) && fs.lstatSync(md_path).isFile()) {
renderer._out_folder = `./upload/${slide}/`;
renderer._in_folder = `./src/${slide}/`;
md = loadMD(md_path, md.meta);
md.meta.series = "";
let model = { title: slide, md: md.md, toc: md.toc, meta: md.meta, pid: slide + "." + build_json.build }
let mdout = `./upload/${slide}/index.html`;
render_one(model, mdout, 'doc');
}
}
fs.ensureDirSync("upload/");
deltaBuild();