From a8fe11b7b520a9b85fc6ffb1848064823eb2e807 Mon Sep 17 00:00:00 2001 From: linguist <22963968+lowlighter@users.noreply.github.com> Date: Wed, 30 Dec 2020 12:33:53 +0100 Subject: [PATCH] Add dynamic indexes --- .github/config/codeql.yml | 4 +-- CONTRIBUTING.md | 3 +- action/index.mjs | 4 +-- src/app.mjs | 5 ++- src/metrics.mjs | 4 +-- src/plugins/index.mjs | 33 ------------------- src/setup.mjs | 55 +++++++++++++++++++++----------- src/templates/common.mjs | 7 +++- src/templates/index.mjs | 13 -------- tests/metrics.test.js | 3 +- utils/build.mjs | 67 --------------------------------------- 11 files changed, 51 insertions(+), 147 deletions(-) delete mode 100644 src/plugins/index.mjs delete mode 100644 src/templates/index.mjs delete mode 100644 utils/build.mjs diff --git a/.github/config/codeql.yml b/.github/config/codeql.yml index bac53a96..346b2e45 100644 --- a/.github/config/codeql.yml +++ b/.github/config/codeql.yml @@ -1,5 +1,3 @@ name: CodeQL config queries: - - uses: security-and-quality -paths-ignore: - - action/dist \ No newline at end of file + - uses: security-and-quality \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8b3efa02..81f49860 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,12 +92,11 @@ It should be avoided when possible as it increases drastically the size of gener * `action.yml` contains GitHub action descriptor * `action/index.mjs` contains GitHub action source code -* `action/dist/index.js` contains compiled the GitHub action code (auto-generated) #### Others * `tests/metrics.mjs` contains tests -* `utils/build.mjs` contains a tool used to rebuild plugins and template indexes and GitHub action +* `utils/build.mjs` contains a tool used to rebuild plugins and template indexes ### 📦 Used packages diff --git a/action/index.mjs b/action/index.mjs index 70dbfe23..8ff615d5 100644 --- a/action/index.mjs +++ b/action/index.mjs @@ -27,7 +27,7 @@ } //Load configuration - const conf = await setup({log:false}) + const {conf, Plugins, Templates} = await setup({log:false}) console.log(`Configuration │ loaded`) console.log(`Version │ ${conf.package.version}`) @@ -201,7 +201,7 @@ q = {...query, ...q, base:false, ...base, ...config, repositories, template} //Render metrics - const rendered = await metrics({login:user, q, dflags}, {graphql, rest, plugins, conf, die}) + const rendered = await metrics({login:user, q, dflags}, {graphql, rest, plugins, conf, die}, {Plugins, Templates}) console.log(`Render │ complete`) //Verify svg diff --git a/src/app.mjs b/src/app.mjs index a5121054..a749e22c 100644 --- a/src/app.mjs +++ b/src/app.mjs @@ -7,14 +7,13 @@ import compression from "compression" import setup from "./setup.mjs" import metrics from "./metrics.mjs" - import Templates from "./templates/index.mjs" import util from "util" /** App */ export default async function () { //Load configuration settings - const conf = await setup() + const {conf, Plugins, Templates} = await setup() const {token, maxusers = 0, restricted = [], debug = false, cached = 30*60*1000, port = 3000, ratelimiter = null, plugins = null} = conf.settings //Load octokits @@ -109,7 +108,7 @@ try { //Render console.debug(`metrics/app/${login} > ${util.inspect(req.query, {depth:Infinity, maxStringLength:256})}`) - const rendered = await metrics({login, q:parse(req.query)}, {graphql, rest, plugins, conf}) + const rendered = await metrics({login, q:parse(req.query)}, {graphql, rest, plugins, conf}, {Plugins, Templates}) //Cache if ((!debug)&&(cached)&&(login !== "placeholder")) cache.put(login, rendered, cached) diff --git a/src/metrics.mjs b/src/metrics.mjs index 7672cf83..e845bf4d 100644 --- a/src/metrics.mjs +++ b/src/metrics.mjs @@ -3,8 +3,6 @@ import SVGO from "svgo" import imgb64 from "image-to-base64" import axios from "axios" - import Plugins from "./plugins/index.mjs" - import Templates from "./templates/index.mjs" import puppeteer from "puppeteer" import url from "url" import processes from "child_process" @@ -14,7 +12,7 @@ import util from "util" //Setup - export default async function metrics({login, q, dflags = []}, {graphql, rest, plugins, conf, die = false}) { + export default async function metrics({login, q, dflags = []}, {graphql, rest, plugins, conf, die = false}, {Plugins, Templates}) { //Compute rendering try { diff --git a/src/plugins/index.mjs b/src/plugins/index.mjs deleted file mode 100644 index dca7181f..00000000 --- a/src/plugins/index.mjs +++ /dev/null @@ -1,33 +0,0 @@ -/* This file is generated automatically with "npm run build" */ - -//Imports - import followup from "./followup/index.mjs" - import gists from "./gists/index.mjs" - import habits from "./habits/index.mjs" - import isocalendar from "./isocalendar/index.mjs" - import languages from "./languages/index.mjs" - import lines from "./lines/index.mjs" - import music from "./music/index.mjs" - import pagespeed from "./pagespeed/index.mjs" - import posts from "./posts/index.mjs" - import projects from "./projects/index.mjs" - import topics from "./topics/index.mjs" - import traffic from "./traffic/index.mjs" - import tweets from "./tweets/index.mjs" - -//Exports - export default { - followup, - gists, - habits, - isocalendar, - languages, - lines, - music, - pagespeed, - posts, - projects, - topics, - traffic, - tweets, - } \ No newline at end of file diff --git a/src/setup.mjs b/src/setup.mjs index 97ac0273..293188fc 100644 --- a/src/setup.mjs +++ b/src/setup.mjs @@ -2,27 +2,38 @@ import fs from "fs" import path from "path" import util from "util" + import url from "url" + const Templates = {} + const Plugins = {} /** Setup */ export default async function ({log = true} = {}) { + //Paths + const __metrics = path.join(path.dirname(url.fileURLToPath(import.meta.url)), "..") + const __templates = path.join(__metrics, "src/templates") + const __plugins = path.join(__metrics, "src/plugins") + const __queries = path.join(__metrics, "src/queries") + const __package = path.join(__metrics, "package.json") + const __settings = path.join(__metrics, "settings.json") + const __statics = path.join(__metrics, "src/html") + const __modules = path.join(__metrics, "node_modules") + //Init const logger = log ? console.debug : () => null logger(`metrics/setup > setup`) - const templates = "src/templates" - const queries = "src/queries" const conf = { templates:{}, queries:{}, settings:{}, - statics:path.resolve("src/html"), - node_modules:path.resolve("node_modules"), + statics:__statics, + node_modules:__modules, } //Load settings logger(`metrics/setup > load settings.json`) - if (fs.existsSync(path.resolve("settings.json"))) { - conf.settings = JSON.parse(`${await fs.promises.readFile(path.resolve("settings.json"))}`) + if (fs.existsSync(__settings)) { + conf.settings = JSON.parse(`${await fs.promises.readFile(__settings)}`) logger(`metrics/setup > load settings.json > success`) } else @@ -37,22 +48,20 @@ //Load package settings logger(`metrics/setup > load package.json`) - conf.package = JSON.parse(`${await fs.promises.readFile(path.resolve("package.json"))}`) + conf.package = JSON.parse(`${await fs.promises.readFile(__package)}`) logger(`metrics/setup > load package.json > success`) //Load templates - for (const name of await fs.promises.readdir(templates)) { - //Cache templates - if (/.*[.]mjs$/.test(name)) + for (const name of await fs.promises.readdir(__templates)) { + //Cache templates file + if (!(await fs.promises.lstat(path.join(__templates, name))).isDirectory()) continue logger(`metrics/setup > load template [${name}]`) - const files = [ - `${templates}/${name}/image.svg`, - `${templates}/${name}/style.css`, - `${templates}/${name}/fonts.css`, - ].map(file => fs.existsSync(path.resolve(file)) ? file : file.replace(`${templates}/${name}/`, `${templates}/classic/`)).map(file => path.resolve(file)) + const files = ["image.svg", "style.css", "fonts.css"].map(file => path.join(__templates, (fs.existsSync(path.join(__templates, name, file)) ? name : "classic"), file)) const [image, style, fonts] = await Promise.all(files.map(async file => `${await fs.promises.readFile(file)}`)) conf.templates[name] = {image, style, fonts} + //Cache templates scripts + Templates[name] = (await import(url.pathToFileURL(path.join(__templates, name, "template.mjs")).href)).default logger(`metrics/setup > load template [${name}] > success`) //Debug if (conf.settings.debug) { @@ -67,19 +76,27 @@ } } + //Load plugins + for (const name of await fs.promises.readdir(__plugins)) { + //Cache plugins scripts + logger(`metrics/setup > load plugin [${name}]`) + Plugins[name] = (await import(url.pathToFileURL(path.join(__plugins, name, "index.mjs")).href)).default + logger(`metrics/setup > load plugin [${name}] > success`) + } + //Load queries - for (const query of await fs.promises.readdir(queries)) { + for (const query of await fs.promises.readdir(__queries)) { //Cache queries const name = query.replace(/[.]graphql$/, "") logger(`metrics/setup > load query [${name}]`) - conf.queries[`_${name}`] = `${await fs.promises.readFile(path.resolve(`${queries}/${query}`))}` + conf.queries[`_${name}`] = `${await fs.promises.readFile(path.join(__queries, query))}` logger(`metrics/setup > load query [${name}] > success`) //Debug if (conf.settings.debug) { Object.defineProperty(conf.queries, `_${name}`, { get() { logger(`metrics/setup > reload query [${name}]`) - const raw = `${fs.readFileSync(path.resolve(`${queries}/${query}`))}` + const raw = `${fs.readFileSync(path.join(__queries, query))}` logger(`metrics/setup > reload query [${name}] > success`) return raw } @@ -97,6 +114,6 @@ //Conf logger(`metrics/setup > setup > success`) - return conf + return {Templates, Plugins, conf} } \ No newline at end of file diff --git a/src/templates/common.mjs b/src/templates/common.mjs index 12753c1c..c429d56c 100644 --- a/src/templates/common.mjs +++ b/src/templates/common.mjs @@ -20,11 +20,16 @@ //Plugins for (const name of Object.keys(imports.plugins)) { + if (!plugins[name].enabled) { + console.log("skipping "+name) + continue + } + pending.push((async () => { try { console.debug(`metrics/compute/${login}/plugins > ${name} > started`) data.plugins[name] = await imports.plugins[name]({login, q, imports, data, computed, rest, graphql, queries}, plugins[name]) - console.debug(`metrics/compute/${login}/plugins > ${name} > completed (${data.plugins[name] !== null ? "success" : "skipped"})`) + console.debug(`metrics/compute/${login}/plugins > ${name} > completed`) } catch (error) { console.debug(`metrics/compute/${login}/plugins > ${name} > completed (error)`) diff --git a/src/templates/index.mjs b/src/templates/index.mjs deleted file mode 100644 index 548d897d..00000000 --- a/src/templates/index.mjs +++ /dev/null @@ -1,13 +0,0 @@ -/* This file is generated automatically with "npm run build" */ - -//Imports - import classic from "./classic/template.mjs" - import repository from "./repository/template.mjs" - import terminal from "./terminal/template.mjs" - -//Exports - export default { - classic, - repository, - terminal, - } \ No newline at end of file diff --git a/tests/metrics.test.js b/tests/metrics.test.js index 294042fa..c7981144 100644 --- a/tests/metrics.test.js +++ b/tests/metrics.test.js @@ -2,9 +2,10 @@ const processes = require("child_process") const yaml = require("js-yaml") const fs = require("fs") + const path = require("path") //Github action - const action = yaml.safeLoad(fs.readFileSync("action.yml", "utf8")) + const action = yaml.safeLoad(fs.readFileSync(path.join(__dirname, "../action.yml"), "utf8")) action.defaults = Object.fromEntries(Object.entries(action.inputs).map(([key, {default:value}]) => [key, /^(yes|no)$/.test(value) ? value === "yes" : value])) action.input = vars => Object.fromEntries([...Object.entries(action.defaults), ...Object.entries(vars)].map(([key, value]) => [`INPUT_${key.toLocaleUpperCase()}`, value])) action.run = async (vars) => await new Promise((solve, reject) => { diff --git a/utils/build.mjs b/utils/build.mjs deleted file mode 100644 index db48e420..00000000 --- a/utils/build.mjs +++ /dev/null @@ -1,67 +0,0 @@ -//Imports - import fs from "fs" - import path from "path" - import url from "url" - import colors from "colors" - -//Initialization - const __dirname = path.join(path.dirname(url.fileURLToPath(import.meta.url)), "..") - const __src = path.join(__dirname, "src") - const __plugins = path.join(__src, "plugins") - const __templates = path.join(__src, "templates") - process.on("unhandledRejection", error => { throw error }) - colors.enable() - -/** Build function */ - export default async function build({actions = ["build"]} = {}) { - //Initialization - const errors = [] - - //Indexes - for (const {name, source, entry} of [{name:"plugins", source:__plugins, entry:"index.mjs"}, {name:"templates", source:__templates, entry:"template.mjs"}]) { - - //Build - const files = (await fs.promises.readdir(source)).filter(name => !/.*[.]mjs$/.test(name)).sort() - const code = [ - `/* This file is generated automatically with "npm run build" */`, - ``, - `//Imports`, - ...files.map(name => ` import ${name} from "./${name}/${entry}"`), - ``, - `//Exports`, - ` export default {`, - ...files.map(name => ` ${name},`), - ` }` - ].join("\n") - console.log(`Generated index for ${name}`.grey) - - //Save build - if (actions.includes("build")) { - fs.promises.writeFile(path.join(source, "index.mjs"), code) - console.log(`Generated index for ${name} saved to ${path.join(source, "index.mjs")}`.green) - } - - //Check build - if (actions.includes("check")) { - const status = `${await fs.promises.readFile(path.join(source, "index.mjs"))}` === code - if (status) - console.log(`Index ${name} is up-to-date`.grey) - else { - console.log(`Index ${name} is outdated`.red) - errors.push(`Index ${name} is outdated, run "npm run build" to fix it`) - } - } - - } - - //Throw on errors - if (errors.length) - throw new Error(`${errors.length} errors occured :\n${errors.map(error => ` - ${error}`).join("\n")}`) - } - -//Main - if (/build.mjs/.test(process.argv[1])) { - //Build - await build() - console.log("Build success !".green) - } \ No newline at end of file