From db603605bd137b322c3f3454dbb1cb8c64fa1ac8 Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Sun, 18 Apr 2021 16:15:17 +0200 Subject: [PATCH] Markdown interpretation (#237) --- package-lock.json | 123 +++++++++++++++++- package.json | 1 + source/app/metrics/index.mjs | 2 +- source/app/metrics/utils.mjs | 25 ++++ source/app/web/instance.mjs | 3 +- source/app/web/statics/about/index.html | 11 +- source/app/web/statics/about/script.js | 5 - source/app/web/statics/index.html | 2 +- source/plugins/activity/index.mjs | 21 +-- source/plugins/stackoverflow/index.mjs | 22 ++-- source/plugins/stackoverflow/metadata.yml | 11 +- .../templates/classic/partials/activity.ejs | 2 +- .../classic/partials/stackoverflow.ejs | 8 +- source/templates/classic/style.css | 53 +++++++- 14 files changed, 240 insertions(+), 49 deletions(-) diff --git a/package-lock.json b/package-lock.json index 34874cf2..297333de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "prismjs": "^1.23.0", "puppeteer": "^8.0.0", "rss-parser": "^3.12.0", + "sanitize-html": "^2.3.3", "simple-git": "^2.37.0", "svgo": "^2.3.0", "twemoji-parser": "^13.0.0", @@ -3282,8 +3283,7 @@ "node_modules/colorette": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.2.tgz", - "integrity": "sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w==", - "dev": true + "integrity": "sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w==" }, "node_modules/combined-stream": { "version": "1.0.8", @@ -3628,7 +3628,6 @@ "version": "4.2.2", "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz", "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -7804,6 +7803,14 @@ "node": ">=6" } }, + "node_modules/klona": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/klona/-/klona-2.0.4.tgz", + "integrity": "sha512-ZRbnvdg/NxqzC7L9Uyqzf4psi1OM4Cuc+sJAkQPjO6XkQIJTNbfK2Rsmbw8fx1p2mkZdp2FZYo2+LwXYY/uwIA==", + "engines": { + "node": ">= 8" + } + }, "node_modules/latest-version": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-5.1.0.tgz", @@ -8221,6 +8228,17 @@ "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==", "devOptional": true }, + "node_modules/nanoid": { + "version": "3.1.22", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.22.tgz", + "integrity": "sha512-/2ZUaJX2ANuLtTvqTlgqBQNJoQO398KyJgZloL0PZkC0dpysjncRUPsFe3DUPzz/y3h+u7C46np8RMuvF3jsSQ==", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -9153,6 +9171,11 @@ "node": ">=8" } }, + "node_modules/parse-srcset": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/parse-srcset/-/parse-srcset-1.0.2.tgz", + "integrity": "sha1-8r0iH2zJcKk42IVWq8WJyqqiveE=" + }, "node_modules/parse5": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz", @@ -9304,6 +9327,23 @@ "node": ">=0.10.0" } }, + "node_modules/postcss": { + "version": "8.2.10", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.2.10.tgz", + "integrity": "sha512-b/h7CPV7QEdrqIxtAf2j31U5ef05uBDuvoXv6L51Q4rcS1jdlXAVKJv+atCFdUXYl9dyTHGyoMzIepwowRJjFw==", + "dependencies": { + "colorette": "^1.2.2", + "nanoid": "^3.1.22", + "source-map": "^0.6.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -10147,6 +10187,31 @@ "node": ">=0.10.0" } }, + "node_modules/sanitize-html": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.3.3.tgz", + "integrity": "sha512-DCFXPt7Di0c6JUnlT90eIgrjs6TsJl/8HYU3KLdmrVclFN4O0heTcVbJiMa23OKVr6aR051XYtsgd8EWwEBwUA==", + "dependencies": { + "deepmerge": "^4.2.2", + "escape-string-regexp": "^4.0.0", + "htmlparser2": "^6.0.0", + "is-plain-object": "^5.0.0", + "klona": "^2.0.3", + "parse-srcset": "^1.0.2", + "postcss": "^8.0.2" + } + }, + "node_modules/sanitize-html/node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", @@ -15070,8 +15135,7 @@ "colorette": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.2.tgz", - "integrity": "sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w==", - "dev": true + "integrity": "sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w==" }, "combined-stream": { "version": "1.0.8", @@ -15350,8 +15414,7 @@ "deepmerge": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz", - "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==", - "dev": true + "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==" }, "defer-to-connect": { "version": "2.0.1", @@ -18704,6 +18767,11 @@ "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", "dev": true }, + "klona": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/klona/-/klona-2.0.4.tgz", + "integrity": "sha512-ZRbnvdg/NxqzC7L9Uyqzf4psi1OM4Cuc+sJAkQPjO6XkQIJTNbfK2Rsmbw8fx1p2mkZdp2FZYo2+LwXYY/uwIA==" + }, "latest-version": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-5.1.0.tgz", @@ -19044,6 +19112,11 @@ "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==", "devOptional": true }, + "nanoid": { + "version": "3.1.22", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.22.tgz", + "integrity": "sha512-/2ZUaJX2ANuLtTvqTlgqBQNJoQO398KyJgZloL0PZkC0dpysjncRUPsFe3DUPzz/y3h+u7C46np8RMuvF3jsSQ==" + }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -19838,6 +19911,11 @@ "lines-and-columns": "^1.1.6" } }, + "parse-srcset": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/parse-srcset/-/parse-srcset-1.0.2.tgz", + "integrity": "sha1-8r0iH2zJcKk42IVWq8WJyqqiveE=" + }, "parse5": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz", @@ -19958,6 +20036,16 @@ "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=", "dev": true }, + "postcss": { + "version": "8.2.10", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.2.10.tgz", + "integrity": "sha512-b/h7CPV7QEdrqIxtAf2j31U5ef05uBDuvoXv6L51Q4rcS1jdlXAVKJv+atCFdUXYl9dyTHGyoMzIepwowRJjFw==", + "requires": { + "colorette": "^1.2.2", + "nanoid": "^3.1.22", + "source-map": "^0.6.1" + } + }, "prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -20654,6 +20742,27 @@ } } }, + "sanitize-html": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.3.3.tgz", + "integrity": "sha512-DCFXPt7Di0c6JUnlT90eIgrjs6TsJl/8HYU3KLdmrVclFN4O0heTcVbJiMa23OKVr6aR051XYtsgd8EWwEBwUA==", + "requires": { + "deepmerge": "^4.2.2", + "escape-string-regexp": "^4.0.0", + "htmlparser2": "^6.0.0", + "is-plain-object": "^5.0.0", + "klona": "^2.0.3", + "parse-srcset": "^1.0.2", + "postcss": "^8.0.2" + }, + "dependencies": { + "escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==" + } + } + }, "sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", diff --git a/package.json b/package.json index 377d24f0..ed965993 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "prismjs": "^1.23.0", "puppeteer": "^8.0.0", "rss-parser": "^3.12.0", + "sanitize-html": "^2.3.3", "simple-git": "^2.37.0", "svgo": "^2.3.0", "twemoji-parser": "^13.0.0", diff --git a/source/app/metrics/index.mjs b/source/app/metrics/index.mjs index 0b1e8f65..ef295773 100644 --- a/source/app/metrics/index.mjs +++ b/source/app/metrics/index.mjs @@ -102,7 +102,7 @@ rendered = await imports.svg.gemojis(rendered, {rest}) //Optimize rendering if (!q.raw) - rendered = xmlformat(rendered, {lineSeparator:"\n"}) + rendered = xmlformat(rendered, {lineSeparator:"\n", collapseContent:true}) if ((conf.settings?.optimize)&&(!q.raw)) { console.debug(`metrics/compute/${login} > optimize`) if (experimental.has("--optimize")) { diff --git a/source/app/metrics/utils.mjs b/source/app/metrics/utils.mjs index ff537adf..88e3e425 100644 --- a/source/app/metrics/utils.mjs +++ b/source/app/metrics/utils.mjs @@ -16,6 +16,11 @@ import nodechartist from "node-chartist" import GIFEncoder from "gifencoder" import PNG from "png-js" + import marked from "marked" + import htmlsanitize from "sanitize-html" + import prism from "prismjs" + import prism_lang from "prismjs/components/index.js" + prism_lang() //Exports export {fs, os, paths, url, util, processes, axios, git, opengraph, jimp, rss} @@ -155,6 +160,26 @@ return false } +/**Markdown-html sanitizer-interpreter */ + export async function markdown(text, {mode = "inline", codelines = Infinity} = {}) { + //Sanitize once user text and then apply markdown. Depending on mode, reapply stricter sanitization if required + let rendered = htmlsanitize(await marked(htmlsanitize(text), { + highlight(code, lang) { + return lang in prism.languages ? prism.highlight(code, prism.languages[lang]) : code + }, + silent:true, + xhtml:true, + }), { + inline:{allowedTags:["br", "code", "span"], allowedAttributes:{code:["class"], span:["class"]}}, + }[mode]) + //Trim code snippets + rendered = rendered.replace(/(?)(?[\s\S]*?)(?<\/code>)/g, (m, open, code, close) => { //eslint-disable-line max-params + const lines = code.trim().split("\n") + return `${open}${lines.slice(0, codelines).join("\n")}${lines.length > codelines ? `\n(${lines.length-codelines} more ${lines.length-codelines === 1 ? "line was" : "lines were"} trimmed)` : ""}${close}` + }) + return rendered + } + /**Image to base64 */ export async function imgb64(image, {width, height, fallback = true} = {}) { //Undefined image diff --git a/source/app/web/instance.mjs b/source/app/web/instance.mjs index 0c5c6b74..6339c4a5 100644 --- a/source/app/web/instance.mjs +++ b/source/app/web/instance.mjs @@ -129,7 +129,6 @@ app.get("/.js/prism.min.js", limiter, (req, res) => res.sendFile(`${conf.paths.node_modules}/prismjs/prism.js`)) app.get("/.js/prism.yaml.min.js", limiter, (req, res) => res.sendFile(`${conf.paths.node_modules}/prismjs/components/prism-yaml.min.js`)) app.get("/.js/prism.markdown.min.js", limiter, (req, res) => res.sendFile(`${conf.paths.node_modules}/prismjs/components/prism-markdown.min.js`)) - app.get("/.js/marked.min.js", limiter, (req, res) => res.sendFile(`${conf.paths.node_modules}/marked/marked.min.js`)) //Meta app.get("/.version", limiter, (req, res) => res.status(200).send(conf.package.version)) app.get("/.requests", limiter, (req, res) => res.status(200).json(requests)) @@ -182,7 +181,7 @@ activity:true, "activity.limit":100, "activity.days":0, notable:true, }, - }, {graphql, rest, plugins:{achievements:{enabled:true}, isocalendar:{enabled:true}, languages:{enabled:true}, activity:{enabled:true}, notable:{enabled:true}}, conf, convert:"json"}, {Plugins, Templates}) + }, {graphql, rest, plugins:{achievements:{enabled:true}, isocalendar:{enabled:true}, languages:{enabled:true}, activity:{enabled:true, markdown:"extended"}, notable:{enabled:true}}, conf, convert:"json"}, {Plugins, Templates}) //Cache if ((!debug)&&(cached)) { const maxage = Math.round(Number(req.query.cache)) diff --git a/source/app/web/statics/about/index.html b/source/app/web/statics/about/index.html index 7c2f4d44..38cf5bc4 100644 --- a/source/app/web/statics/about/index.html +++ b/source/app/web/statics/about/index.html @@ -190,7 +190,7 @@