From 3c4402e4ba82b6ce89d21eba3adf38487139381c Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Fri, 24 Jun 2022 22:35:09 +0200 Subject: [PATCH] refactor(metrics/insights): new features (#1098) --- source/app/metrics/index.mjs | 88 +++--- source/app/web/instance.mjs | 59 +++- source/app/web/statics/about/index.html | 365 +++++++++++++++++++----- source/app/web/statics/about/script.js | 101 ++++++- source/app/web/statics/about/style.css | 236 ++++++++++++++- source/plugins/base/index.mjs | 8 +- source/plugins/core/examples.yml | 8 + source/plugins/core/index.mjs | 3 +- source/plugins/reactions/index.mjs | 5 +- source/plugins/topics/index.mjs | 1 + 10 files changed, 732 insertions(+), 142 deletions(-) diff --git a/source/app/metrics/index.mjs b/source/app/metrics/index.mjs index 722d306b..bb7713fd 100644 --- a/source/app/metrics/index.mjs +++ b/source/app/metrics/index.mjs @@ -4,7 +4,7 @@ import util from "util" import * as utils from "./utils.mjs" //Setup -export default async function metrics({login, q}, {graphql, rest, plugins, conf, die = false, verify = false, convert = null}, {Plugins, Templates}) { +export default async function metrics({login, q}, {graphql, rest, plugins, conf, die = false, verify = false, convert = null, callbacks = null}, {Plugins, Templates}) { //Compute rendering try { //Debug @@ -59,8 +59,8 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, //Executing base plugin and compute metrics console.debug(`metrics/compute/${login} > compute`) - await Plugins.base({login, q, data, rest, graphql, plugins, queries, pending, imports}, conf) - await computer({login, q}, {conf, data, rest, graphql, plugins, queries, account: data.account, convert, template}, {pending, imports}) + await Plugins.base({login, q, data, rest, graphql, plugins, queries, pending, imports, callbacks}, conf) + await computer({login, q}, {conf, data, rest, graphql, plugins, queries, account: data.account, convert, template, callbacks}, {pending, imports}) const promised = await Promise.all(pending) //Check plugins errors @@ -211,39 +211,53 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, } //Metrics insights -metrics.insights = async function({login}, {graphql, rest, conf}, {Plugins, Templates}) { - const q = { - template: "classic", - achievements: true, - "achievements.threshold": "X", - isocalendar: true, - "isocalendar.duration": "full-year", - languages: true, - "languages.limit": 0, - activity: true, - "activity.limit": 100, - "activity.days": 0, - notable: true, - followup: true, - "followup.sections": "repositories, user", - habits: true, - "habits.from": 100, - "habits.days": 7, - "habits.facts": false, - "habits.charts": true, - introduction: true, - } - const plugins = { - achievements: {enabled: true}, - isocalendar: {enabled: true}, - languages: {enabled: true, extras: false}, - activity: {enabled: true, markdown: "extended"}, - notable: {enabled: true}, - followup: {enabled: true}, - habits: {enabled: true, extras: false}, - introduction: {enabled: true}, - } - return metrics({login, q}, {graphql, rest, plugins, conf, convert: "json"}, {Plugins, Templates}) +metrics.insights = async function({login}, {graphql, rest, conf, callbacks}, {Plugins, Templates}) { + return metrics({login, q:metrics.insights.q}, {graphql, rest, plugins:metrics.insights.plugins, conf, callbacks, convert: "json"}, {Plugins, Templates}) +} +metrics.insights.q = { + template: "classic", + achievements: true, + "achievements.threshold": "X", + isocalendar: true, + "isocalendar.duration": "full-year", + languages: true, + "languages.limit": 0, + activity: true, + "activity.limit": 100, + "activity.days": 0, + "activity.timestamps": true, + notable: true, + "notable.repositories": true, + followup: true, + "followup.sections": "repositories, user", + introduction: true, + topics: true, + "topics.mode": "icons", + "topics.limit": 0, + stars: true, + "stars.limit": 6, + reactions: true, + "reactions.details": "percentage", + repositories: true, + "repositories.pinned": 6, + sponsors: true, + calendar: true, + "calendar.limit": 0, +} +metrics.insights.plugins = { + achievements: {enabled: true}, + isocalendar: {enabled: true}, + languages: {enabled: true, extras: false}, + activity: {enabled: true, markdown: "extended"}, + notable: {enabled: true}, + followup: {enabled: true}, + introduction: {enabled: true}, + topics: {enabled: true}, + stars: {enabled: true}, + reactions: {enabled: true}, + repositories: {enabled: true}, + sponsors: {enabled: true}, + calendar: {enabled: true}, } //Metrics insights static render @@ -278,5 +292,5 @@ metrics.insights.output = async function({login, imports, conf}, {graphql, rest, ` await browser.close() - return {mime: "text/html", rendered} + return {mime: "text/html", rendered, errors:json.errors} } diff --git a/source/app/web/instance.mjs b/source/app/web/instance.mjs index b5e78cbb..6a7d09d2 100644 --- a/source/app/web/instance.mjs +++ b/source/app/web/instance.mjs @@ -172,11 +172,32 @@ export default async function({sandbox = false} = {}) { } }) + //Pending requests + const pending = new Map() + //About routes app.use("/about/.statics/", express.static(`${conf.paths.statics}/about`)) app.get("/about/", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/about/index.html`)) app.get("/about/index.html", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/about/index.html`)) app.get("/about/:login", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/about/index.html`)) + app.get("/about/query/:login/:plugin/", async (req, res) => { + //Check username + const login = req.params.login?.replace(/[\n\r]/g, "") + if (!/^[-\w]+$/i.test(login)) { + console.debug(`metrics/app/${login}/insights > 400 (invalid username)`) + return res.status(400).send("Bad request: username seems invalid") + } + //Check plugin + const plugin = req.params.plugin?.replace(/[\n\r]/g, "") + if (!/^\w+$/i.test(plugin)) { + console.debug(`metrics/app/${login}/insights > 400 (invalid plugin name)`) + return res.status(400).send("Bad request: plugin name seems invalid") + } + if (cache.get(`about.${login}.${plugin}`)) { + return res.send(cache.get(`about.${login}.${plugin}`)) + } + return res.status(204).send("No content: no data fetched yet") + }) app.get("/about/query/:login/", ...middlewares, async (req, res) => { //Check username const login = req.params.login?.replace(/[\n\r]/g, "") @@ -185,7 +206,16 @@ export default async function({sandbox = false} = {}) { return res.status(400).send("Bad request: username seems invalid") } //Compute metrics + let solve = null try { + //Prevent multiples requests + if ((!debug) && (!mock) && (pending.has(`about.${login}`))) { + console.debug(`metrics/app/${login}/insights > awaiting pending request`) + await pending.get(`about.${login}`) + } + else { + pending.set(`about.${login}`, new Promise(_solve => solve = _solve)) + } //Read cached data if possible if ((!debug) && (cached) && (cache.get(`about.${login}`))) { console.debug(`metrics/app/${login}/insights > using cached results`) @@ -193,13 +223,28 @@ export default async function({sandbox = false} = {}) { } //Compute metrics console.debug(`metrics/app/${login}/insights > compute insights`) - const json = await metrics.insights({login}, {graphql, rest, conf}, {Plugins, Templates}) - //Cache - if ((!debug) && (cached)) { - const maxage = Math.round(Number(req.query.cache)) - cache.put(`about.${login}`, json, maxage > 0 ? maxage : cached) + const callbacks = { + async plugin(login, plugin, success, result) { + console.debug(`metrics/app/${login}/insights/plugins > ${plugin} > ${success ? "success" : "failure"}`) + cache.put(`about.${login}.${plugin}`, result) + } } - return res.json(json) + ;(async () => { + try { + const json = await metrics.insights({login}, {graphql, rest, conf, callbacks}, {Plugins, Templates}) + //Cache + cache.put(`about.${login}`, json) + if ((!debug) && (cached)) { + const maxage = Math.round(Number(req.query.cache)) + cache.put(`about.${login}`, json, maxage > 0 ? maxage : cached) + } + } + catch (error) { + console.error(`metrics/app/${login}/insights > error > ${error}`) + } + })() + console.debug(`metrics/app/${login}/insights > accepted request`) + return res.status(202).json({processing:true, plugins:Object.keys(metrics.insights.plugins)}) } //Internal error catch (error) { @@ -219,12 +264,12 @@ export default async function({sandbox = false} = {}) { return res.status(500).send("Internal Server Error: failed to process metrics correctly") } finally { + solve?.() _requests_refresh = true } }) //Metrics - const pending = new Map() app.get("/:login/:repository?", ...middlewares, async (req, res) => { //Request params const login = req.params.login?.replace(/[\n\r]/g, "") diff --git a/source/app/web/statics/about/index.html b/source/app/web/statics/about/index.html index 6af21be1..be095837 100644 --- a/source/app/web/statics/about/index.html +++ b/source/app/web/statics/about/index.html @@ -22,6 +22,10 @@ Metrics Insights {{ version }} +
+
+
+
@@ -81,55 +85,17 @@
-
+
{{ introduction }}
-
-
-
- - - - - - - - - - - - -
-
-
{{ format("number", leaderboard.user) }}{{ {1:"st", 2:"nd", 3:"rd"}[leaderboard.user%10] ?? "th" }}
-
/ {{ format("number", leaderboard.total, {notation:"compact", compactDisplay:"long"}) }} {{ leaderboard.type }}
-
{{ prefix }} {{ prefix.length ? title.toLocaleLowerCase() : title }}
-
{{ text }}
-
-
-
- -
-

- - Notable contributions -

-
- - - @{{ name }} - -
+
+
Fetching achievements
+ + +
+

+ + Sponsor {{ user }} +

+
+
+ + +
+
+ +
+

+ + Notable contributions +

+
Fetching data
+ +
+ No contributions to display +
+
+ +
+

+ + Featured repositories +

+
Fetching data
+
+
+
+ + +
+ {{ repository.nameWithOwner }} + created {{ repository.created }} + starred on {{ format("date", repository.starredAt) }} +
+
+
{{ repository.description }}
+
+
+ + {{ repository.primaryLanguage.name }} +
+
+ + {{ repository.licenseInfo.name ?? repository.licenseInfo.spdxId }} +
+
+ + {{ format("number", repository.stargazerCount) }} +
+
+ + {{ format("number", repository.forkCount) }} +
+
+ + {{ format("number", repository.issues.totalCount) }} +
+
+ + {{ format("number", repository.pullRequests.totalCount) }} +
+
+
+
+
+ No featured repositories +
+
+ +
+

+ + Overall users reactions +

+
Fetching data
+ +
+ No users reactions +
+
+ +
+
+

+ + Starred topics +

+
Fetching data
+ +
+ No starred topics. +
+

- Languages used + Used languages

-
-
-
-
- {{ name }} - ({{ format("number", value, {style:"percent", maximumFractionDigits:2})}}) +
Fetching data
+
No languages used
-
+
+
+

+ + Commits history +

+
Fetching isometric calendar
+
+
+
Fetching data
+ +
+ +

Overall status of related issues and pull requests

-
+
Fetching data
+
-
- -
-
-

- - Commits calendar -

-
+
+ No available data
-
+

- - Average commits at each hour over the last week + + Recently starred repositories

-
-
- {{ habits[h] }} -
- {{ `${h}`.padStart(2, 0) }} +
Fetching data
+
+
+
+ + +
+ {{ repository.nameWithOwner }} + created {{ repository.created }} + starred on {{ format("date", repository.starredAt) }} +
+
+
{{ repository.description }}
+
+
+ + {{ repository.primaryLanguage.name }} +
+
+ + {{ repository.licenseInfo.name ?? repository.licenseInfo.spdxId }} +
+
+ + {{ format("number", repository.stargazerCount) }} +
+
+ + {{ format("number", repository.forkCount) }} +
+
+ + {{ format("number", repository.issues.totalCount) }} +
+
+ + {{ format("number", repository.pullRequests.totalCount) }} +
+
+
+ No recently starred repositories +
@@ -254,7 +464,8 @@ Recent activity -