diff --git a/source/plugins/languages/analyzers.mjs b/source/plugins/languages/analyzers.mjs new file mode 100644 index 00000000..d3bfc33c --- /dev/null +++ b/source/plugins/languages/analyzers.mjs @@ -0,0 +1,164 @@ +/**Indepth analyzer */ +export async function indepth({login, data, imports}, {skipped}) { + //Check prerequisites + if (!await imports.which("github-linguist")) + throw new Error("Feature requires github-linguist") + + //Compute repositories stats from fetched repositories + const results = {total:0, stats:{}} + for (const repository of data.user.repositories.nodes) { + //Skip repository if asked + if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) { + console.debug(`metrics/compute/${login}/plugins > languages > skipped repository ${repository.owner.login}/${repository.name}`) + continue + } + + //Repository handle + const repo = `${repository.owner.login}/${repository.name}` + console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking ${repo}`) + + //Temporary directory + const path = imports.paths.join(imports.os.tmpdir(), `${data.user.databaseId}-${repo.replace(/[^\w]/g, "_")}`) + console.debug(`metrics/compute/${login}/plugins > languages > indepth > cloning ${repo} to temp dir ${path}`) + + //Process + try { + //Git clone into temporary directory + await imports.fs.rmdir(path, {recursive:true}) + await imports.fs.mkdir(path, {recursive:true}) + const git = await imports.git(path) + await git.clone(`https://github.com/${repo}`, ".").status() + + //Analyze repository + await analyze(arguments[0], {results, path}) + } + catch { + console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured while processing ${repo}, skipping...`) + } + finally { + //Cleaning + console.debug(`metrics/compute/${login}/plugins > languages > indepth > cleaning temp dir ${path}`) + await imports.fs.rmdir(path, {recursive:true}) + } + } + return results +} + +/**Recent languages activity */ +export async function recent({login, data, imports, rest, account}, {skipped}) { + //Check prerequisites + if (!await imports.which("github-linguist")) + throw new Error("Feature requires github-linguist") + + //Get user recent activity + console.debug(`metrics/compute/${login}/plugins > languages > querying api`) + const commits = [], days = 14, pages = 3, results = {total:0, stats:{}} + try { + for (let page = 1; page <= pages; page++) { + console.debug(`metrics/compute/${login}/plugins > languages > loading page ${page}`) + commits.push(...(await rest.activity.listEventsForAuthenticatedUser({username:login, per_page:100, page})).data + .filter(({type}) => type === "PushEvent") + .filter(({actor}) => account === "organization" ? true : actor.login === login) + .filter(({repo:{name:repo}}) => (!skipped.includes(repo.toLocaleLowerCase())) && (!skipped.includes(repo.toLocaleLowerCase().split("/").pop()))) + .filter(({created_at}) => new Date(created_at) > new Date(Date.now() - days * 24 * 60 * 60 * 1000)) + ) + } + } + catch { + console.debug(`metrics/compute/${login}/plugins > languages > no more page to load`) + } + console.debug(`metrics/compute/${login}/plugins > languages > ${commits.length} commits loaded`) + + //Retrieve edited files and filter edited lines (those starting with +/-) from patches + console.debug(`metrics/compute/${login}/plugins > languages > loading patches`) + const patches = [ + ...await Promise.allSettled( + commits + .flatMap(({payload}) => payload.commits).map(commit => commit.url) + .map(async commit => (await rest.request(commit)).data.files), + ), + ] + .filter(({status}) => status === "fulfilled") + .map(({value}) => value) + .flatMap(files => files.map(file => ({name:imports.paths.basename(file.filename), patch:file.patch ?? ""}))) + .map(({name, patch}) => ({name, patch:patch.split("\n").filter(line => /^[+]/.test(line)).map(line => line.substring(1)).join("\n")})) + + //Temporary directory + const path = imports.paths.join(imports.os.tmpdir(), `${data.user.databaseId}`) + console.debug(`metrics/compute/${login}/plugins > languages > creating temp dir ${path} with ${patches.length} files`) + + //Process + try { + //Save patches in temporary directory + await imports.fs.rmdir(path, {recursive:true}) + await imports.fs.mkdir(path, {recursive:true}) + await Promise.all(patches.map(({name, patch}, i) => imports.fs.writeFile(imports.paths.join(path, `${i}${imports.paths.extname(name)}`), patch))) + + //Create temporary git repository + console.debug(`metrics/compute/${login}/plugins > languages > creating temp git repository`) + const git = await imports.git(path) + await git.init().add(".").addConfig("user.name", login).addConfig("user.email", "<>").commit("linguist").status() + + //Analyze repository + await analyze(arguments[0], {results, path}) + } + catch { + console.debug(`metrics/compute/${login}/plugins > languages > an error occured while processing recently used languages`) + } + finally { + //Cleaning + console.debug(`metrics/compute/${login}/plugins > languages > cleaning temp dir ${path}`) + await imports.fs.rmdir(path, {recursive:true}) + } + + console.log(results) + return results +} + +/**Analyze a single repository */ +async function analyze({login, imports}, {results, path}) { + //Spawn linguist process and map files to languages + console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`) + const files = Object.fromEntries(Object.entries(JSON.parse(await imports.run("github-linguist --json", {cwd:path}, {log:false}))).flatMap(([lang, files]) => files.map(file => [file, lang]))) + + console.log(files) + + //Processing diff + const per_page = 10 + console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking git log`) + for (let page = 0; ; page++) { + try { + const stdout = await imports.run(`git log --author="${login}" --format="" --patch --max-count=${per_page} --skip=${page*per_page}`, {cwd:path}, {log:false}) + let file = null, lang = null + if (!stdout.trim().length) { + console.debug(`metrics/compute/${login}/plugins > languages > indepth > no more commits`) + break + } + console.debug(`metrics/compute/${login}/plugins > languages > indepth > processing commits ${page*per_page} from ${(page+1)*per_page}`) + for (const line of stdout.split("\n").map(line => line.trim())) { + //Ignore empty lines or unneeded lines + if ((!/^[+]/.test(line))||(!line.length)) + continue + //File marker + if (/^[+]{3}\sb[/](?[\s\S]+)$/.test(line)) { + file = line.match(/^[+]{3}\sb[/](?[\s\S]+)$/)?.groups?.file ?? null + lang = files[file] ?? null + continue + } + //Ignore unkonwn languages + if (!lang) + continue + //Added line marker + if (/^[+]\s(?[\s\S]+)$/.test(line)) { + const size = Buffer.byteLength(line.match(/^[+]\s(?[\s\S]+)$/)?.groups?.line ?? "", "utf-8") + results.stats[lang] = (results.stats[lang] ?? 0) + size + results.total += size + } + } + } + catch { + console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured on page ${page}, skipping...`) + } + } + +} \ No newline at end of file diff --git a/source/plugins/languages/indepth.mjs b/source/plugins/languages/indepth.mjs deleted file mode 100644 index c03b6ae2..00000000 --- a/source/plugins/languages/indepth.mjs +++ /dev/null @@ -1,105 +0,0 @@ -/**Indepth analyzer */ -export default async function({login, data, imports}, {skipped, ignored}) { - //Check prerequisites - if (!await imports.which("github-linguist")) - throw new Error("Feature requires github-linguist") - - //Compute repositories stats - const results = {total:0, stats:{}} - for (const repository of data.user.repositories.nodes) { - const repo = `${repository.owner.login}/${repository.name}` - console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking ${repo}`) - //Skip repository if asked - if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) { - console.debug(`metrics/compute/${login}/plugins > languages > skipped repository ${repository.owner.login}/${repository.name}`) - continue - } - //Analyze - try { - await analyze(arguments[0], {repo, results}) - } - catch { - console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured while processing ${repo}, skipping...`) - } - } - - //Ignore languages if asked - Object.assign(results.stats, Object.fromEntries(Object.entries(results.stats).filter(([lang]) => !ignored.includes(lang.toLocaleLowerCase())))) - return results -} - -/**Clone and analyze a single repository */ -async function analyze({login, data, imports}, {repo, results}) { - //Git clone into a temporary directory - const path = imports.paths.join(imports.os.tmpdir(), `${data.user.databaseId}-${repo.replace(/[^\w]/g, "_")}`) - console.debug(`metrics/compute/${login}/plugins > languages > indepth > cloning ${repo} to temp dir ${path}`) - await imports.fs.rmdir(path, {recursive:true}) - await imports.fs.mkdir(path, {recursive:true}) - const git = await imports.git(path) - await git.clone(`https://github.com/${repo}`, ".").status() - - //Spawn linguist process and map files to languages - console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`) - const files = {} - { - const stdout = await imports.run("github-linguist --breakdown", {cwd:path}, {log:false}) - let lang = null - for (const line of stdout.split("\n").map(line => line.trim())) { - //Ignore empty lines - if (!line.length) - continue - //Language marker - if (/^(?[\s\S]+):\s*$/.test(line)) { - lang = line.match(/^(?[\s\S]+):\s*$/)?.groups?.lang ?? null - continue - } - //Store language - if (lang) { - files[line] = {lang} - continue - } - } - } - - //Processing diff - const per_page = 10 - console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking git log`) - for (let page = 0; ; page++) { - try { - const stdout = await imports.run(`git log --author="${login}" --format="" --patch --max-count=${per_page} --skip=${page*per_page}`, {cwd:path}, {log:false}) - let file = null, lang = null - if (!stdout.trim().length) { - console.debug(`metrics/compute/${login}/plugins > languages > indepth > no more commits`) - break - } - console.debug(`metrics/compute/${login}/plugins > languages > indepth > processing commits ${page*per_page} from ${(page+1)*per_page}`) - for (const line of stdout.split("\n").map(line => line.trim())) { - //Ignore empty lines or unneeded lines - if ((!/^[+]/.test(line))||(!line.length)) - continue - //File marker - if (/^[+]{3}\sb[/](?[\s\S]+)$/.test(line)) { - file = line.match(/^[+]{3}\sb[/](?[\s\S]+)$/)?.groups?.file ?? null - lang = files[file]?.lang ?? null - continue - } - //Ignore unkonwn languages - if (!lang) - continue - //Added line marker - if (/^[+]\s(?[\s\S]+)$/.test(line)) { - const size = Buffer.byteLength(line.match(/^[+]\s(?[\s\S]+)$/)?.groups?.line ?? "", "utf-8") - results.stats[lang] = (results.stats[lang] ?? 0) + size - results.total += size - } - } - } - catch { - console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured on page ${page}, skipping...`) - } - } - - //Cleaning - console.debug(`metrics/compute/${login}/plugins > languages > indepth > cleaning temp dir ${path}`) - await imports.fs.rmdir(path, {recursive:true}) -} \ No newline at end of file diff --git a/source/plugins/languages/index.mjs b/source/plugins/languages/index.mjs index 526e879b..3d196694 100644 --- a/source/plugins/languages/index.mjs +++ b/source/plugins/languages/index.mjs @@ -1,8 +1,8 @@ //Imports -import indepth_analyzer from "./indepth.mjs" +import { indepth as indepth_analyzer, recent as recent_analyzer } from "./analyzers.mjs" //Setup -export default async function({login, data, imports, q, account}, {enabled = false} = {}) { +export default async function({login, data, imports, q, rest, account}, {enabled = false} = {}) { //Plugin execution try { //Check if plugin is enabled and requirements are met @@ -10,7 +10,7 @@ export default async function({login, data, imports, q, account}, {enabled = fal return null //Load inputs - let {ignored, skipped, colors, details, threshold, limit, indepth} = imports.metadata.plugins.languages.inputs({data, account, q}) + let {ignored, skipped, colors, details, threshold, limit, indepth, sections} = imports.metadata.plugins.languages.inputs({data, account, q}) threshold = (Number(threshold.replace(/%$/, "")) || 0) / 100 skipped.push(...data.shared["repositories.skipped"]) if (!limit) @@ -25,7 +25,7 @@ export default async function({login, data, imports, q, account}, {enabled = fal //Iterate through user's repositories and retrieve languages data console.debug(`metrics/compute/${login}/plugins > languages > processing ${data.user.repositories.nodes.length} repositories`) - const languages = {details, colors:{}, total:0, stats:{}} + const languages = {sections, details, colors:{}, total:0, stats:{}, "stats.recent":{}} for (const repository of data.user.repositories.nodes) { //Skip repository if asked if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) { @@ -34,33 +34,35 @@ export default async function({login, data, imports, q, account}, {enabled = fal } //Process repository languages for (const {size, node:{color, name}} of Object.values(repository.languages.edges)) { - //Ignore language if asked - if (ignored.includes(name.toLocaleLowerCase())) { - console.debug(`metrics/compute/${login}/plugins > languages > ignored language ${name}`) - continue - } - //Update language stats languages.stats[name] = (languages.stats[name] ?? 0) + size languages.colors[name] = colors[name.toLocaleLowerCase()] ?? color ?? "#ededed" languages.total += size } } + //Recently used languages + if ((sections.includes("recently-used"))&&(["user", "organization"].includes(account))) { + console.debug(`metrics/compute/${login}/plugins > languages > using recent analyzer`) + languages["stats.recent"] = await recent_analyzer({login, data, imports, rest, account}, {skipped}) + } + //Indepth mode if (indepth) { console.debug(`metrics/compute/${login}/plugins > languages > switching to indepth mode (this may take some time)`) - Object.assign(languages, await indepth_analyzer({login, data, imports}, {skipped, ignored})) + Object.assign(languages, await indepth_analyzer({login, data, imports}, {skipped})) } //Compute languages stats - console.debug(`metrics/compute/${login}/plugins > languages > computing stats`) - languages.favorites = Object.entries(languages.stats).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size:value, color:languages.colors[name], x:0})).filter(({value}) => value / languages.total > threshold) - const visible = {total:Object.values(languages.favorites).map(({size}) => size).reduce((a, b) => a + b, 0)} - for (let i = 0; i < languages.favorites.length; i++) { - languages.favorites[i].value /= visible.total - languages.favorites[i].x = (languages.favorites[i - 1]?.x ?? 0) + (languages.favorites[i - 1]?.value ?? 0) - if ((colors[i]) && (!colors[languages.favorites[i].name.toLocaleLowerCase()])) - languages.favorites[i].color = colors[i] + for (const {section, stats = {}, total = 0} of [{section:"favorites", stats:languages.stats, total:languages.total}, {section:"recent", ...languages["stats.recent"]}]) { + console.debug(`metrics/compute/${login}/plugins > languages > computing stats ${section}`) + languages[section] = Object.entries(stats).filter(([name]) => !ignored.includes(name.toLocaleLowerCase())).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size:value, color:languages.colors[name], x:0})).filter(({value}) => value / total > threshold) + const visible = {total:Object.values(languages[section]).map(({size}) => size).reduce((a, b) => a + b, 0)} + for (let i = 0; i < languages[section].length; i++) { + languages[section][i].value /= visible.total + languages[section][i].x = (languages[section][i - 1]?.x ?? 0) + (languages[section][i - 1]?.value ?? 0) + if ((colors[i]) && (!colors[languages[section][i].name.toLocaleLowerCase()])) + languages[section][i].color = colors[i] + } } //Results diff --git a/source/plugins/languages/metadata.yml b/source/plugins/languages/metadata.yml index 9d191847..ae91b53e 100644 --- a/source/plugins/languages/metadata.yml +++ b/source/plugins/languages/metadata.yml @@ -39,6 +39,17 @@ inputs: min: 0 max: 8 + # Sections to display + plugin_languages_sections: + description: Sections to display + type: array + format: comma-separated + default: most-used + example: most-used, recently-used + values: + - most-used # Most used languages + - recently-used # Recently used languages + # Overrides default languages colors # Use `${n}:${color}` to change the color of the n-th most used language (e.g. "0:red" to make your most used language red) # Use `${language}:${color}` to change the color of named language (e.g. "javascript:red" to make JavaScript language red, language case is ignored) diff --git a/source/templates/classic/partials/languages.ejs b/source/templates/classic/partials/languages.ejs index f1bc1c87..e0f902a5 100644 --- a/source/templates/classic/partials/languages.ejs +++ b/source/templates/classic/partials/languages.ejs @@ -1,8 +1,8 @@ -<% if (plugins.languages) { %> +<% if (plugins.languages) { for (const section of plugins.languages.sections) { const languages = {"most-used":plugins.languages.favorites, "recently-used":plugins.languages.recent}[section] %>

- Most used languages + <%= {"most-used":"Most used languages", "recently-used":"Recently used languages"}[section] %>

<% if (plugins.languages.error) { %>
@@ -16,8 +16,8 @@ - - <% for (const {name, value, color, x} of plugins.languages.favorites) { %> + + <% for (const {name, value, color, x} of languages) { %> <% } %> @@ -25,7 +25,7 @@
<% for (const row of rows) { %>
- <% for (const {name, value, color, size} of plugins.languages.favorites.filter((_, i) => i%rows.length === row)) { %> + <% for (const {name, value, color, size} of languages.filter((_, i) => i%rows.length === row)) { %>
@@ -42,7 +42,7 @@
<% } else { %>
- <% for (const {name, value, color} of plugins.languages.favorites) { %> + <% for (const {name, value, color} of languages) { %>
<%= name %> @@ -52,4 +52,4 @@ <% } %> <% } %>
-<% } %> \ No newline at end of file +<% } } %> \ No newline at end of file