From 22d442a03c9929349e6e6f36cd5a5e5350d795f0 Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Tue, 25 May 2021 20:41:53 +0200 Subject: [PATCH] Add indepth languages analysis (#325) --- source/app/metrics/utils.mjs | 8 +- source/plugins/languages/README.md | 11 +++ source/plugins/languages/indepth.mjs | 105 ++++++++++++++++++++++++++ source/plugins/languages/index.mjs | 11 ++- source/plugins/languages/metadata.yml | 9 ++- 5 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 source/plugins/languages/indepth.mjs diff --git a/source/app/metrics/utils.mjs b/source/app/metrics/utils.mjs index e29d0ed8..76c0dd9d 100644 --- a/source/app/metrics/utils.mjs +++ b/source/app/metrics/utils.mjs @@ -131,7 +131,7 @@ export async function chartist() { } /**Run command */ -export async function run(command, options, {prefixed = true} = {}) { +export async function run(command, options, {prefixed = true, log = true} = {}) { const prefix = {win32:"wsl"}[process.platform] ?? "" command = `${prefixed ? prefix : ""} ${command}`.trim() return new Promise((solve, reject) => { @@ -142,8 +142,10 @@ export async function run(command, options, {prefixed = true} = {}) { child.stderr.on("data", data => stderr += data) child.on("close", code => { console.debug(`metrics/command > ${command} > exited with code ${code}`) - console.debug(stdout) - console.debug(stderr) + if (log) { + console.debug(stdout) + console.debug(stderr) + } return code === 0 ? solve(stdout) : reject(stderr) }) }) diff --git a/source/plugins/languages/README.md b/source/plugins/languages/README.md index e9bb8467..a85517c6 100644 --- a/source/plugins/languages/README.md +++ b/source/plugins/languages/README.md @@ -17,6 +17,16 @@ You can specify either an index with a color, or a language name (case insensiti Colors can be either in hexadecimal format or a [named color](https://developer.mozilla.org/en-US/docs/Web/CSS/color_value). It is also possible to use a predefined set of colors from [colorsets.json](colorsets.json) +**Using `indepth` statistics** + +Languages statistics are computed using the top languages of each repository you contributed to. +If you work a lot with other people, these numbers may be less representative of your actual work. + +The `plugin_languages_indepth` option lets you get more accurate metrics by cloning each repository, running [github/linguist](https://github.com/github/linguist) on it and iterating over patches matching your username from `git log`, but will be **significantly slower**. + +> ⚠️ Although *metrics* does not send any code to external sources, you must understand that when using this option repositories are cloned locally temporarly on the GitHub Action runner. If you work with sensitive data or company code, it is advised to keep this option disabled. *Metrics* cannot be held responsible for any eventual code leaks, use at your own risk. +> Source code is available for auditing at [indepth.mjs](/source/plugins/languages/indepth.mjs) + #### ℹ️ Examples workflows [➡️ Available options for this plugin](metadata.yml) @@ -32,4 +42,5 @@ It is also possible to use a predefined set of colors from [colorsets.json](colo plugin_languages_details: bytes-size, percentage # Additionally display total bytes size and percentage plugin_languages_threshold: 2% # Hides all languages less than 2% plugin_languages_limit: 8 # Display up to 8 languages + plugin_languages_indepth: no # Get indepth stats (see documentation before enabling) ``` diff --git a/source/plugins/languages/indepth.mjs b/source/plugins/languages/indepth.mjs new file mode 100644 index 00000000..c03b6ae2 --- /dev/null +++ b/source/plugins/languages/indepth.mjs @@ -0,0 +1,105 @@ +/**Indepth analyzer */ +export default async function({login, data, imports}, {skipped, ignored}) { + //Check prerequisites + if (!await imports.which("github-linguist")) + throw new Error("Feature requires github-linguist") + + //Compute repositories stats + const results = {total:0, stats:{}} + for (const repository of data.user.repositories.nodes) { + const repo = `${repository.owner.login}/${repository.name}` + console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking ${repo}`) + //Skip repository if asked + if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) { + console.debug(`metrics/compute/${login}/plugins > languages > skipped repository ${repository.owner.login}/${repository.name}`) + continue + } + //Analyze + try { + await analyze(arguments[0], {repo, results}) + } + catch { + console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured while processing ${repo}, skipping...`) + } + } + + //Ignore languages if asked + Object.assign(results.stats, Object.fromEntries(Object.entries(results.stats).filter(([lang]) => !ignored.includes(lang.toLocaleLowerCase())))) + return results +} + +/**Clone and analyze a single repository */ +async function analyze({login, data, imports}, {repo, results}) { + //Git clone into a temporary directory + const path = imports.paths.join(imports.os.tmpdir(), `${data.user.databaseId}-${repo.replace(/[^\w]/g, "_")}`) + console.debug(`metrics/compute/${login}/plugins > languages > indepth > cloning ${repo} to temp dir ${path}`) + await imports.fs.rmdir(path, {recursive:true}) + await imports.fs.mkdir(path, {recursive:true}) + const git = await imports.git(path) + await git.clone(`https://github.com/${repo}`, ".").status() + + //Spawn linguist process and map files to languages + console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`) + const files = {} + { + const stdout = await imports.run("github-linguist --breakdown", {cwd:path}, {log:false}) + let lang = null + for (const line of stdout.split("\n").map(line => line.trim())) { + //Ignore empty lines + if (!line.length) + continue + //Language marker + if (/^(?[\s\S]+):\s*$/.test(line)) { + lang = line.match(/^(?[\s\S]+):\s*$/)?.groups?.lang ?? null + continue + } + //Store language + if (lang) { + files[line] = {lang} + continue + } + } + } + + //Processing diff + const per_page = 10 + console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking git log`) + for (let page = 0; ; page++) { + try { + const stdout = await imports.run(`git log --author="${login}" --format="" --patch --max-count=${per_page} --skip=${page*per_page}`, {cwd:path}, {log:false}) + let file = null, lang = null + if (!stdout.trim().length) { + console.debug(`metrics/compute/${login}/plugins > languages > indepth > no more commits`) + break + } + console.debug(`metrics/compute/${login}/plugins > languages > indepth > processing commits ${page*per_page} from ${(page+1)*per_page}`) + for (const line of stdout.split("\n").map(line => line.trim())) { + //Ignore empty lines or unneeded lines + if ((!/^[+]/.test(line))||(!line.length)) + continue + //File marker + if (/^[+]{3}\sb[/](?[\s\S]+)$/.test(line)) { + file = line.match(/^[+]{3}\sb[/](?[\s\S]+)$/)?.groups?.file ?? null + lang = files[file]?.lang ?? null + continue + } + //Ignore unkonwn languages + if (!lang) + continue + //Added line marker + if (/^[+]\s(?[\s\S]+)$/.test(line)) { + const size = Buffer.byteLength(line.match(/^[+]\s(?[\s\S]+)$/)?.groups?.line ?? "", "utf-8") + results.stats[lang] = (results.stats[lang] ?? 0) + size + results.total += size + } + } + } + catch { + console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured on page ${page}, skipping...`) + } + } + + //Cleaning + console.debug(`metrics/compute/${login}/plugins > languages > indepth > cleaning temp dir ${path}`) + await imports.fs.rmdir(path, {recursive:true}) +} \ No newline at end of file diff --git a/source/plugins/languages/index.mjs b/source/plugins/languages/index.mjs index c5a8ee6a..526e879b 100644 --- a/source/plugins/languages/index.mjs +++ b/source/plugins/languages/index.mjs @@ -1,3 +1,6 @@ +//Imports +import indepth_analyzer from "./indepth.mjs" + //Setup export default async function({login, data, imports, q, account}, {enabled = false} = {}) { //Plugin execution @@ -7,7 +10,7 @@ export default async function({login, data, imports, q, account}, {enabled = fal return null //Load inputs - let {ignored, skipped, colors, details, threshold, limit} = imports.metadata.plugins.languages.inputs({data, account, q}) + let {ignored, skipped, colors, details, threshold, limit, indepth} = imports.metadata.plugins.languages.inputs({data, account, q}) threshold = (Number(threshold.replace(/%$/, "")) || 0) / 100 skipped.push(...data.shared["repositories.skipped"]) if (!limit) @@ -43,6 +46,12 @@ export default async function({login, data, imports, q, account}, {enabled = fal } } + //Indepth mode + if (indepth) { + console.debug(`metrics/compute/${login}/plugins > languages > switching to indepth mode (this may take some time)`) + Object.assign(languages, await indepth_analyzer({login, data, imports}, {skipped, ignored})) + } + //Compute languages stats console.debug(`metrics/compute/${login}/plugins > languages > computing stats`) languages.favorites = Object.entries(languages.stats).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size:value, color:languages.colors[name], x:0})).filter(({value}) => value / languages.total > threshold) diff --git a/source/plugins/languages/metadata.yml b/source/plugins/languages/metadata.yml index 548a1d9e..9d191847 100644 --- a/source/plugins/languages/metadata.yml +++ b/source/plugins/languages/metadata.yml @@ -69,4 +69,11 @@ inputs: plugin_languages_threshold: description: Minimum threshold type: string - default: 0% \ No newline at end of file + default: 0% + + # Compute indepth languages statistics by cloning repositories and processing your commits individually + # See documentation before enabling + plugin_languages_indepth: + description: Indepth languages processing (see documentation before enabling) + type: boolean + default: false \ No newline at end of file