688 lines
19 KiB
JavaScript
688 lines
19 KiB
JavaScript
const { PerformanceObserver, performance } = require('perf_hooks');
|
|
let t_start = performance.now();
|
|
|
|
process.on('exit', function () {
|
|
let t_end = performance.now();
|
|
console.log("Done in " + (t_end - t_start) + "ms");
|
|
});
|
|
|
|
const fs = require('fs-extra');
|
|
const glob = require('glob-fs')({ gitignore: true });
|
|
const marked = require('marked');
|
|
const marked_toc = require('marked');
|
|
const plantuml = require('plantuml');
|
|
const _ = require('lodash');
|
|
const slugify = require('uslug');
|
|
const { transformFml } = require('./fmlTransformer')
|
|
|
|
|
|
const fml_syntax_highlight = require('./fml_syntax_highlight');
|
|
|
|
const child_process = require('child_process');
|
|
|
|
|
|
function run(cmd, cwd, done) {
|
|
|
|
console.log(cmd);
|
|
|
|
const exec = child_process.exec;
|
|
exec(cmd, { cwd }, (err, stdout, stderr) => {
|
|
if (err) {
|
|
//some err occurred
|
|
console.error(err);
|
|
} else {
|
|
// the *entire* stdout and stderr (buffered)
|
|
console.log(`stdout: ${stdout}`);
|
|
console.log(`stderr: ${stderr}`);
|
|
if (typeof done == "function") {
|
|
done(err, stdout, stderr);
|
|
} else { }
|
|
}
|
|
});
|
|
|
|
}
|
|
|
|
|
|
const build_json = fs.readJSONSync('build.json', { throws: false }) || {};
|
|
|
|
build_json.doc_build = (build_json.doc_build || 0) + 1;
|
|
|
|
fs.outputJsonSync('build.json', build_json, { spaces: 2 });
|
|
|
|
|
|
String.prototype.hashCode = function () {
|
|
var hash = 0, i, chr;
|
|
if (this.length === 0) return hash;
|
|
for (i = 0; i < this.length; i++) {
|
|
chr = this.charCodeAt(i);
|
|
hash = ((hash << 5) - hash) + chr;
|
|
hash |= 0; // Convert to 32bit integer
|
|
}
|
|
return hash;
|
|
};
|
|
|
|
const cache_json = fs.readJSONSync('./upload/puml_cache.json', { throws: false }) || {};
|
|
|
|
|
|
function cache_lookup(text, miss) {
|
|
let hash = new String(text).hashCode();
|
|
|
|
if (cache_json[hash]) { //hit
|
|
return new Promise((resolve, reject) => resolve(cache_json[hash]));
|
|
} else {
|
|
return miss(text).then((output) => {
|
|
cache_json[hash] = output;
|
|
fs.outputJson('./upload/puml_cache.json', cache_json, { spaces: 2 });
|
|
return output;
|
|
})
|
|
}
|
|
}
|
|
|
|
|
|
let puml_count = 0;
|
|
let epuml_count = 0;
|
|
let fml_count = 0;
|
|
|
|
|
|
function toc(str) {
|
|
|
|
// var defaultTemplate = '<%= depth %><%= bullet %>[<%= heading %>](#<%= url %>)\n';
|
|
var defaultTemplate = '<a class="d<%=d%>" href="#<%= url %>"><%- heading %></a>\n';
|
|
|
|
const utils = {};
|
|
|
|
utils.arrayify = function (arr) {
|
|
return !Array.isArray(arr) ? [arr] : arr;
|
|
};
|
|
|
|
utils.escapeRegex = function (re) {
|
|
return re.replace(/(\[|\]|\(|\)|\/|\.|\^|\$|\*|\+|\?)/g, '\\$1');
|
|
};
|
|
|
|
utils.isDest = function (dest) {
|
|
return !dest || dest === 'undefined' || typeof dest === 'object';
|
|
};
|
|
|
|
utils.isMatch = function (keys, str) {
|
|
keys = utils.arrayify(keys);
|
|
keys = (keys.length > 0) ? keys.join('|') : '.*';
|
|
|
|
// Escape certain characters, like '[', '('
|
|
var k = utils.escapeRegex(String(keys));
|
|
|
|
// Build up the regex to use for replacement patterns
|
|
var re = new RegExp('(?:' + k + ')', 'g');
|
|
if (String(str).match(re)) {
|
|
return true;
|
|
} else {
|
|
return false;
|
|
}
|
|
};
|
|
|
|
utils.sanitize = function (src) {
|
|
src = src.replace(/(\s*\[!|(?:\[.+ →\]\()).+/g, '');
|
|
src = src.replace(/\s*\*\s*\[\].+/g, '');
|
|
return src;
|
|
};
|
|
|
|
utils.slugify = function (str) {
|
|
str = str.replace(/\/\//g, '-');
|
|
str = str.replace(/\//g, '-');
|
|
str = str.replace(/\./g, '-');
|
|
str = _.str.slugify(str);
|
|
str = str.replace(/^-/, '');
|
|
str = str.replace(/-$/, '');
|
|
return str;
|
|
};
|
|
|
|
var omit = ['grunt', 'helper', 'handlebars-helper', 'mixin', 'filter', 'assemble-contrib', 'assemble'];
|
|
|
|
utils.strip = function (name, options) {
|
|
var opts = _.extend({}, options);
|
|
if (opts.omit === false) { omit = []; }
|
|
var exclusions = _.union(omit, utils.arrayify(opts.strip || []));
|
|
var re = new RegExp('^(?:' + exclusions.join('|') + ')[-_]?', 'g');
|
|
return name.replace(re, '');
|
|
};
|
|
|
|
var opts = {
|
|
firsth1: false,
|
|
blacklist: true,
|
|
omit: [],
|
|
maxDepth: 3,
|
|
slugifyOptions: { allowedChars: '-' },
|
|
slugify: function (text) {
|
|
return slugify(text, opts.slugifyOptions).replace("ex-", "ex---");
|
|
}
|
|
};
|
|
|
|
var toc = '';
|
|
var tokens = marked.lexer(str);
|
|
var tocArray = [];
|
|
|
|
// Remove the very first h1, true by default
|
|
if (opts.firsth1 === false) {
|
|
tokens.shift();
|
|
}
|
|
|
|
// Do any h1's still exist?
|
|
var h1 = _.some(tokens, { depth: 1 });
|
|
|
|
var task_count = 0;
|
|
|
|
tokens.filter(function (token) {
|
|
// Filter out everything but headings
|
|
if (token.type !== 'heading' || token.type === 'code') {
|
|
return false;
|
|
}
|
|
|
|
// Since we removed the first h1, we'll check to see if other h1's
|
|
// exist. If none exist, then we unindent the rest of the TOC
|
|
if (!h1) {
|
|
token.depth = token.depth - 1;
|
|
}
|
|
|
|
// Store original text and create an id for linking
|
|
token.heading = opts.strip ? utils.strip(token.text, opts) : token.text;
|
|
|
|
if (token.heading.indexOf("Ex -") >= 0) {
|
|
token.heading = token.heading.replace("Ex -", `<span class='task'>${task_count}.</span> `);
|
|
task_count++;
|
|
if (task_count === 13) task_count++; // skip hands-on
|
|
}
|
|
|
|
// Create a "slugified" id for linking
|
|
token.id = opts.slugify(token.text);
|
|
|
|
// Omit headings with these strings
|
|
var omissions = ['Table of Contents', 'TOC', 'TABLE OF CONTENTS'];
|
|
var omit = _.union([], opts.omit, omissions);
|
|
|
|
if (utils.isMatch(omit, token.heading)) {
|
|
return;
|
|
}
|
|
|
|
return true;
|
|
}).forEach(function (h) {
|
|
|
|
if (h.depth > opts.maxDepth) {
|
|
return;
|
|
}
|
|
|
|
var bullet = Array.isArray(opts.bullet)
|
|
? opts.bullet[(h.depth - 1) % opts.bullet.length]
|
|
: opts.bullet;
|
|
|
|
var data = _.extend({}, opts.data, {
|
|
d: h.depth,
|
|
depth: new Array((h.depth - 1) * 2 + 1).join(' '),
|
|
bullet: bullet ? bullet : '* ',
|
|
heading: h.heading,
|
|
url: h.id
|
|
});
|
|
|
|
tocArray.push(data);
|
|
toc += ejs.render(defaultTemplate, data);
|
|
//toc += _.template(opts.template || defaultTemplate, data);
|
|
});
|
|
|
|
return {
|
|
data: tocArray,
|
|
toc: opts.strip
|
|
? utils.strip(toc, opts)
|
|
: toc
|
|
};
|
|
}
|
|
|
|
|
|
|
|
const renderer = new marked.Renderer();
|
|
|
|
renderer._paragraph = renderer.paragraph;
|
|
|
|
renderer.paragraph = function (text) {
|
|
|
|
//console.log(text);
|
|
|
|
if (text.trim().startsWith("{")) {
|
|
|
|
renderer.sec_depth = renderer.sec_depth || 0;
|
|
renderer.sec_depth++;
|
|
|
|
let c = text.split(".");
|
|
c.shift();
|
|
|
|
if (c.length > 0) {
|
|
return `<section class="${c.join(" ")}">`;
|
|
} else {
|
|
return "<section>"
|
|
}
|
|
}
|
|
|
|
if (text.trim().startsWith("}")) {
|
|
|
|
renderer.sec_depth = renderer.sec_depth || 0;
|
|
renderer.sec_depth--;
|
|
|
|
return "</section>"
|
|
}
|
|
|
|
if (text.trim().startsWith("Pass: ") && renderer.sec_depth > 0) {
|
|
return `<section class="pass"><span>★☆☆: </span>${text.split("Pass: ")[1]}</section>`;
|
|
}
|
|
if (text.trim().startsWith("Good: ") && renderer.sec_depth > 0) {
|
|
return `<section class="ok"><span>★★☆: </span>${text.split("Good: ")[1]}</section>`;
|
|
}
|
|
if (text.trim().startsWith("Exceed: ") && renderer.sec_depth > 0) {
|
|
return `<section class="exceed"><span>★★★: </span>${text.split("Exceed: ")[1]}</section>`;
|
|
}
|
|
if (text.trim().startsWith("Hint: ") && renderer.sec_depth > 0) {
|
|
return `<section class="hint"><span>Hint: </span>${text.split("Hint: ")[1]}</section>`;
|
|
}
|
|
|
|
return renderer._paragraph(text);
|
|
}
|
|
|
|
renderer._code = renderer.code;
|
|
|
|
renderer.code = function (code, infostring, escaped) {
|
|
// console.log(code, infostring, escaped);
|
|
if (infostring) {
|
|
if (infostring.startsWith("puml")) {
|
|
let i = epuml_count++;
|
|
cache_lookup(code, () => plantuml(code)).then(svg => {
|
|
fs.writeFile(renderer._out_folder + "puml_e" + i + ".svg", svg)
|
|
})
|
|
return `<img src="./puml_e${i}.svg">`;
|
|
}
|
|
|
|
if (infostring.startsWith("fml1") || infostring.startsWith("fml2")) {
|
|
|
|
let fml = "" + code;
|
|
|
|
let i = fml_count++;
|
|
|
|
const fml_output = renderer._out_folder + "fml_" + i + ".svg";
|
|
const tempFile = renderer._out_folder + `fml_${i}.temp.fml`;
|
|
|
|
try{
|
|
fs.ensureFileSync(tempFile);
|
|
|
|
const transformedFml = transformFml(code, infostring.startsWith("fml1") ? "Flat" : "Nested")
|
|
|
|
fs.writeFile(tempFile, transformedFml)
|
|
.then(_ => {
|
|
run("node fml " + tempFile + " " + fml_output, undefined, () => {
|
|
fs.remove(tempFile)
|
|
})
|
|
})
|
|
|
|
if (infostring.indexOf("?src") >= 0) {
|
|
|
|
return `<aside class="flex"><pre><code>${highlight_fml(fml)}</code></pre><img src="./fml_${i}.svg"></aside>`;
|
|
|
|
} else {
|
|
|
|
return `<img src="./fml_${i}.svg">`;
|
|
|
|
}
|
|
|
|
} catch(e) {
|
|
//the original code will be printed (opportunity to show the error)
|
|
console.log(e);
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
return renderer._code(code, infostring, escaped);
|
|
|
|
};
|
|
|
|
//TODO (["'])(?:(?=(\\?))\2.)*?\1 for quoted strings (which do not contain keywords)
|
|
function highlight_fml(s) {
|
|
return fml_syntax_highlight.parse(s);
|
|
let keywords = ["feature", "optional", "required", "hasOneOf", "hasSome", "has", "requires", "excludes"] //fml1
|
|
|
|
return keywords.reduce((s, k) => s.replace(new RegExp(k+" ", "g"), `<b>${k}</b> `, s)
|
|
.replace(new RegExp(k+"\n", "g"), `<b>${k}</b>\n`, s)
|
|
.replace(new RegExp(k, "g"), `<b>${k}</b>`), s);
|
|
}
|
|
|
|
renderer._image = renderer.image;
|
|
|
|
renderer.image = function (href, title, text) {
|
|
|
|
//console.log(href, title, text);
|
|
|
|
let url = new URL(href, "http://localhost/");
|
|
|
|
href = "." + url.pathname;
|
|
|
|
if (href.endsWith(".puml")) {
|
|
|
|
let i = puml_count++;
|
|
|
|
fs.readFile(href.replace("./", renderer._in_folder)).then(puml => {
|
|
return cache_lookup(puml, () => plantuml(puml))
|
|
//return plantuml(puml)
|
|
}).then(svg => {
|
|
fs.writeFile(renderer._out_folder + "puml_" + i + ".svg", svg)
|
|
})
|
|
|
|
return `<img src="./puml_${i}.svg" alt="${text}">`;
|
|
|
|
}
|
|
|
|
if (href.endsWith(".fml1") || href.endsWith(".fml2")) {
|
|
|
|
let i = fml_count++;
|
|
|
|
const fml_output = renderer._out_folder + "fml_" + i + ".svg";
|
|
const tempFile = renderer._out_folder + `fml_${i}.temp.fml`;
|
|
|
|
fs.ensureFileSync(tempFile);
|
|
|
|
let fml = fs.readFileSync(href.replace("./", renderer._in_folder), 'utf8');
|
|
|
|
const transformedFml = transformFml(fml, href.endsWith(".fml1") ? "Flat" : "Nested")
|
|
|
|
fs.writeFile(tempFile, transformedFml)
|
|
.then(_ => {
|
|
run("node fml " + tempFile + " " + fml_output, undefined, () => {
|
|
fs.remove(tempFile)
|
|
})
|
|
})
|
|
|
|
if (url.searchParams.has("src")) {
|
|
|
|
return `<aside class="flex"><pre><code>${highlight_fml(fml)}</code></pre><img src="./fml_${i}.svg" alt="${text}"></aside>`;
|
|
|
|
} else {
|
|
|
|
return `<img src="./fml_${i}.svg" alt="${text}">`;
|
|
|
|
}
|
|
}
|
|
|
|
if (href.endsWith(".fml") || href.endsWith(".fml0")) {
|
|
|
|
let i = fml_count++;
|
|
|
|
const fml_output = renderer._out_folder + "fml_" + i + ".svg";
|
|
|
|
run("node fml " + href.replace("./", renderer._in_folder) + " " + fml_output);
|
|
|
|
return `<img src="./fml_${i}.svg" alt="${text}">`;
|
|
|
|
}
|
|
|
|
if (href.endsWith(".c5")) {
|
|
|
|
let i = fml_count++;
|
|
|
|
const fml_output = renderer._out_folder + "c5_" + i + ".svg";
|
|
|
|
run("node c5 " + href.replace("./", renderer._in_folder) + " " + fml_output);
|
|
|
|
return `<img src="./c5_${i}.svg" alt="${text}">`;
|
|
|
|
}
|
|
|
|
if (href.endsWith(".madr") || href.endsWith(".md")) {
|
|
|
|
let md = fs.readFileSync(href.replace("./", renderer._in_folder));
|
|
|
|
return `<h3>${text}</h3>` + marked.parse("" + new String(md));
|
|
|
|
}
|
|
|
|
if (href.endsWith(".svg") || href.endsWith(".png") ) {
|
|
let fileName = href.split("/").slice(-1);
|
|
let out = renderer._out_folder + `${fileName}`;
|
|
fs.copySync(href.replace("./", renderer._in_folder), out);
|
|
|
|
return `<img src="${fileName}" alt="${text}">`;
|
|
}
|
|
|
|
return renderer._image(href, title, text);
|
|
|
|
}
|
|
|
|
renderer._heading = renderer.heading;
|
|
|
|
let task_count = 0;
|
|
|
|
renderer.heading = function (text, level, raw, slugger) {
|
|
|
|
// console.log(text);
|
|
|
|
if (text.indexOf("Ex -") >= 0) {
|
|
text = text.replace("Ex -", `<span class='task'>${task_count}.</span> `);
|
|
task_count++;
|
|
if (task_count === 13) task_count++; // skip hands-on
|
|
}
|
|
|
|
// let sec = "<section>"
|
|
|
|
// if (task_count > 1) {
|
|
// sec = "</section>" + sec;
|
|
// }
|
|
|
|
return renderer._heading(text, level, raw, slugger);
|
|
};
|
|
|
|
|
|
marked.setOptions({
|
|
renderer, //: new marked.Renderer(),
|
|
highlight: function (code) {
|
|
return code; //require('highlight').highlightAuto(code).value;
|
|
},
|
|
pedantic: false,
|
|
gfm: true,
|
|
breaks: false,
|
|
sanitize: false,
|
|
smartLists: true,
|
|
smartypants: false,
|
|
xhtml: false
|
|
});
|
|
|
|
function loadMD(f, vars = {}) {
|
|
|
|
let txt = "" + new String(fs.readFileSync(f));
|
|
|
|
if (txt.indexOf("---") >= 0) {
|
|
|
|
let parts = txt.split("\n---\n");
|
|
|
|
parts.slice(0, -1).forEach(p => {
|
|
let lines = p.split("\n");
|
|
let k = lines[0].split(":");
|
|
vars[k[0]] = lines.slice(1).join("\n");
|
|
});
|
|
|
|
txt = parts[parts.length - 1];
|
|
|
|
}
|
|
|
|
//convert strings to booleans
|
|
Object.keys(vars).forEach(k => {
|
|
if (vars[k] === "false") vars[k] = false;
|
|
if (vars[k] === "true") vars[k] = true;
|
|
})
|
|
|
|
console.log(vars);
|
|
|
|
task_count = 0;
|
|
|
|
return { md: marked.parse(txt), meta: vars, toc: toc(txt) };
|
|
}
|
|
|
|
|
|
let ejs = require('ejs');
|
|
|
|
let views = {}
|
|
|
|
function render_one(model, out_file, template) {
|
|
|
|
model.out_file = out_file;
|
|
|
|
views[template] = views[template] || new String(fs.readFileSync("./template/" + template + ".ejs"));
|
|
|
|
let html = ejs.render(views[template], model);
|
|
|
|
fs.ensureFileSync(out_file);
|
|
fs.writeFileSync(out_file, html);
|
|
|
|
};
|
|
|
|
|
|
var walk = function (dir, done) {
|
|
var results = [];
|
|
fs.readdir(dir, function (err, list) {
|
|
if (err) return done(err);
|
|
var i = 0;
|
|
(function next() {
|
|
var file = list[i++];
|
|
if (!file) return done(null, results);
|
|
file = dir + '/' + file;
|
|
fs.stat(file, function (err, stat) {
|
|
if (stat && stat.isDirectory()) {
|
|
walk(file, function (err, res) {
|
|
results = results.concat(res);
|
|
next();
|
|
});
|
|
} else {
|
|
results.push({ name: file, stat });
|
|
next();
|
|
}
|
|
});
|
|
})();
|
|
});
|
|
};
|
|
|
|
|
|
|
|
function deltaBuild() {
|
|
let old_file_list = {};
|
|
try {
|
|
old_file_list = fs.readJsonSync('upload/files.json');
|
|
} catch (e) { }
|
|
|
|
walk("src", function (err, list) {
|
|
|
|
if (list === undefined) {
|
|
console.log("Missing src folder")
|
|
return;
|
|
}
|
|
|
|
if (list.length == 0) {
|
|
console.log("Empty src folder")
|
|
return;
|
|
}
|
|
|
|
let flat = {};
|
|
list.filter((f) => !f.name.endsWith(".DS_Store")).forEach(f => {
|
|
flat[f.name] = f.stat;
|
|
});
|
|
|
|
let todo = {};
|
|
|
|
let deleted = {};
|
|
Object.keys(old_file_list).forEach(k => { deleted[k] = old_file_list[k] });
|
|
|
|
let changed = [];
|
|
|
|
function extract_dir(k) {
|
|
let a = k.split("/")
|
|
return a.slice(1, 3).join("/");
|
|
}
|
|
|
|
function push(k) {
|
|
let d = extract_dir(k);
|
|
todo[d] = (todo[d] || 0) + 1;
|
|
|
|
changed.push(k);
|
|
}
|
|
|
|
Object.keys(flat).forEach((k) => {
|
|
if (old_file_list[k]) {
|
|
if (flat[k].mtimeMs != old_file_list[k].mtimeMs) {
|
|
push(k);
|
|
}
|
|
delete deleted[k]; //k was found so what's left over should have been deleted
|
|
} else {
|
|
push(k); //this is a new addition
|
|
}
|
|
});
|
|
|
|
fs.writeJson("upload/files.json", flat);
|
|
|
|
console.log('Changed', changed);
|
|
console.log('Deleted', Object.keys(deleted));
|
|
|
|
if (Object.keys(todo).length == 0) {
|
|
todo['sa/model'] = 1;
|
|
}
|
|
|
|
//todo['exercises/8-ws'] = 1;
|
|
|
|
if (Object.keys(todo).indexOf('templates/doc.ejs') != -1) {
|
|
todo = Array.from(new Set(Object.keys(flat).map(extract_dir))).reduce((a, c) => { a[c] = 1; return a; }, {});
|
|
delete todo['templates/doc.ejs'];
|
|
}
|
|
|
|
Object.keys(todo).filter(k => k.endsWith('meta.json')).forEach(mk => {
|
|
delete todo[mk];
|
|
Array.from(new Set(Object.keys(flat).map(extract_dir))).filter(k => k.startsWith(mk.split("/")[0])).forEach(k => {
|
|
todo[k] = 1;
|
|
})
|
|
})
|
|
|
|
console.log("Building", todo);
|
|
|
|
Object.keys(todo).forEach(buildDoc.bind(null, changed, Object.keys(deleted)));
|
|
|
|
})
|
|
}
|
|
|
|
function buildDoc(changed, deleted, slide) {
|
|
|
|
console.log("Building " + slide);
|
|
|
|
let md_path = "./src/" + slide + "/index.md";
|
|
|
|
let md = { meta: {} };
|
|
|
|
let meta_path = "./src/" + slide.split("/")[0] + "/meta.json";
|
|
try {
|
|
md.meta = fs.readJSONSync(meta_path);
|
|
} catch (e) {
|
|
//console.log(e);
|
|
}
|
|
|
|
md.meta.build = build_json.doc_build;
|
|
//md.meta.lecture = "Web Atelier 2020 ";
|
|
md.meta.timestamp = new Date().toLocaleString();
|
|
|
|
if (fs.existsSync(md_path) && fs.lstatSync(md_path).isFile()) {
|
|
|
|
renderer._out_folder = `./upload/${slide}/`;
|
|
renderer._in_folder = `./src/${slide}/`;
|
|
|
|
md = loadMD(md_path, md.meta);
|
|
|
|
md.meta.series = "";
|
|
|
|
let model = { title: slide, md: md.md, toc: md.toc, meta: md.meta, pid: slide + "." + build_json.build }
|
|
|
|
let mdout = `./upload/${slide}/index.html`;
|
|
|
|
render_one(model, mdout, 'doc');
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fs.ensureDirSync("upload/");
|
|
deltaBuild(); |