From 9c4b709f3626d8201aea0b923ad8df6378303c42 Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Wed, 26 May 2021 22:07:09 +0200 Subject: [PATCH] Improvements languages indepth analysis (#329) --- .../api/github/graphql/base.repositories.mjs | 6 + source/plugins/base/index.mjs | 20 ++-- .../plugins/base/queries/repositories.graphql | 2 +- source/plugins/languages/analyzers.mjs | 17 ++- source/plugins/languages/index.mjs | 8 +- source/plugins/languages/metadata.yml | 1 + .../templates/classic/partials/languages.ejs | 7 +- source/templates/classic/style.css | 3 + .../repository/partials/languages.ejs | 103 +++++++++--------- 9 files changed, 91 insertions(+), 76 deletions(-) diff --git a/source/app/mocks/api/github/graphql/base.repositories.mjs b/source/app/mocks/api/github/graphql/base.repositories.mjs index 2a9d6c8b..73de521a 100644 --- a/source/app/mocks/api/github/graphql/base.repositories.mjs +++ b/source/app/mocks/api/github/graphql/base.repositories.mjs @@ -4,6 +4,9 @@ export default function({faker, query, login = faker.internet.userName()}) { return /after: "MOCKED_CURSOR"/m.test(query) ? ({ user:{ + get repositoriesContributedTo() { + return this.repositories + }, repositories:{ edges:[], nodes:[], @@ -12,6 +15,9 @@ export default function({faker, query, login = faker.internet.userName()}) { }) : ({ user:{ + get repositoriesContributedTo() { + return this.repositories + }, repositories:{ edges:[ { diff --git a/source/plugins/base/index.mjs b/source/plugins/base/index.mjs index 16ede239..054e0d05 100644 --- a/source/plugins/base/index.mjs +++ b/source/plugins/base/index.mjs @@ -34,22 +34,24 @@ export default async function({login, graphql, data, q, queries, imports}, conf) Object.assign(data, {user:queried[account]}) postprocess?.[account]({login, data}) //Query repositories from GitHub API - { + data.user.repositoriesContributedTo.nodes = data.user.repositoriesContributedTo.nodes ?? [] + for (const type of ["repositories", "repositoriesContributedTo"]) { //Iterate through repositories let cursor = null let pushed = 0 + const options = {repositories:{forks, affiliations, constraints:""}, repositoriesContributedTo:{forks:"", affiliations:"", constraints:", includeUserRepositories: false, contributionTypes: COMMIT"}}[type] ?? null do { - console.debug(`metrics/compute/${login}/base > retrieving repositories after ${cursor}`) - const {[account]:{repositories:{edges, nodes}}} = await graphql(queries.base.repositories({login, account, after:cursor ? `after: "${cursor}"` : "", repositories:Math.min(repositories, {user:100, organization:25}[account]), forks, affiliations})) + console.debug(`metrics/compute/${login}/base > retrieving ${type} after ${cursor}`) + const {[account]:{[type]:{edges = [], nodes = []} = {}}} = await graphql(queries.base.repositories({login, account, type, after:cursor ? `after: "${cursor}"` : "", repositories:Math.min(repositories, {user:100, organization:25}[account]), ...options})) cursor = edges?.[edges?.length - 1]?.cursor - data.user.repositories.nodes.push(...nodes) + data.user[type].nodes.push(...nodes) pushed = nodes.length - console.debug(`metrics/compute/${login}/base > retrieved ${pushed} repositories after ${cursor}`) - } while ((pushed) && (cursor) && (data.user.repositories.nodes.length < repositories)) + console.debug(`metrics/compute/${login}/base > retrieved ${pushed} ${type} after ${cursor}`) + } while ((pushed) && (cursor) && (data.user.repositories.nodes.length + data.user.repositoriesContributedTo.nodes.length < repositories)) //Limit repositories - console.debug(`metrics/compute/${login}/base > keeping only ${repositories} repositories`) - data.user.repositories.nodes.splice(repositories) - console.debug(`metrics/compute/${login}/base > loaded ${data.user.repositories.nodes.length} repositories`) + console.debug(`metrics/compute/${login}/base > keeping only ${repositories} ${type}`) + data.user[type].nodes.splice(repositories) + console.debug(`metrics/compute/${login}/base > loaded ${data.user[type].nodes.length} ${type}`) } //Success console.debug(`metrics/compute/${login}/base > graphql query > account ${account} > success`) diff --git a/source/plugins/base/queries/repositories.graphql b/source/plugins/base/queries/repositories.graphql index 5174edf8..a8e471c0 100644 --- a/source/plugins/base/queries/repositories.graphql +++ b/source/plugins/base/queries/repositories.graphql @@ -1,6 +1,6 @@ query BaseRepositories { $account(login: "$login") { - repositories($after first: $repositories $forks $affiliations, orderBy: {field: UPDATED_AT, direction: DESC}) { + $type($after first: $repositories $forks $affiliations $constraints, orderBy: {field: UPDATED_AT, direction: DESC}) { edges { cursor } diff --git a/source/plugins/languages/analyzers.mjs b/source/plugins/languages/analyzers.mjs index d3bfc33c..13a2d14e 100644 --- a/source/plugins/languages/analyzers.mjs +++ b/source/plugins/languages/analyzers.mjs @@ -1,12 +1,12 @@ /**Indepth analyzer */ -export async function indepth({login, data, imports}, {skipped}) { +export async function indepth({login, data, imports, repositories}, {skipped}) { //Check prerequisites if (!await imports.which("github-linguist")) throw new Error("Feature requires github-linguist") //Compute repositories stats from fetched repositories - const results = {total:0, stats:{}} - for (const repository of data.user.repositories.nodes) { + const results = {total:0, lines:{}, stats:{}} + for (const repository of repositories) { //Skip repository if asked if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) { console.debug(`metrics/compute/${login}/plugins > languages > skipped repository ${repository.owner.login}/${repository.name}`) @@ -52,7 +52,7 @@ export async function recent({login, data, imports, rest, account}, {skipped}) { //Get user recent activity console.debug(`metrics/compute/${login}/plugins > languages > querying api`) - const commits = [], days = 14, pages = 3, results = {total:0, stats:{}} + const commits = [], days = 14, pages = 3, results = {total:0, lines:{}, stats:{}} try { for (let page = 1; page <= pages; page++) { console.debug(`metrics/compute/${login}/plugins > languages > loading page ${page}`) @@ -110,8 +110,6 @@ export async function recent({login, data, imports, rest, account}, {skipped}) { console.debug(`metrics/compute/${login}/plugins > languages > cleaning temp dir ${path}`) await imports.fs.rmdir(path, {recursive:true}) } - - console.log(results) return results } @@ -121,8 +119,6 @@ async function analyze({login, imports}, {results, path}) { console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`) const files = Object.fromEntries(Object.entries(JSON.parse(await imports.run("github-linguist --json", {cwd:path}, {log:false}))).flatMap(([lang, files]) => files.map(file => [file, lang]))) - console.log(files) - //Processing diff const per_page = 10 console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking git log`) @@ -149,9 +145,10 @@ async function analyze({login, imports}, {results, path}) { if (!lang) continue //Added line marker - if (/^[+]\s(?[\s\S]+)$/.test(line)) { - const size = Buffer.byteLength(line.match(/^[+]\s(?[\s\S]+)$/)?.groups?.line ?? "", "utf-8") + if (/^[+]\s*(?[\s\S]+)$/.test(line)) { + const size = Buffer.byteLength(line.match(/^[+]\s*(?[\s\S]+)$/)?.groups?.line ?? "", "utf-8") results.stats[lang] = (results.stats[lang] ?? 0) + size + results.lines[lang] = (results.lines[lang] ?? 0) + 1 results.total += size } } diff --git a/source/plugins/languages/index.mjs b/source/plugins/languages/index.mjs index f78e567a..ed12dabb 100644 --- a/source/plugins/languages/index.mjs +++ b/source/plugins/languages/index.mjs @@ -31,7 +31,8 @@ export default async function({login, data, imports, q, rest, account}, {enabled console.debug(`metrics/compute/${login}/plugins > languages > custom colors ${JSON.stringify(colors)}`) //Unique languages - const unique = new Set(data.user.repositories.nodes.flatMap(repository => repository.languages.edges.map(({node:{name}}) => name))).size + const repositories = [...data.user.repositories.nodes, ...data.user.repositoriesContributedTo.nodes] + const unique = new Set(repositories.flatMap(repository => repository.languages.edges.map(({node:{name}}) => name))).size //Iterate through user's repositories and retrieve languages data console.debug(`metrics/compute/${login}/plugins > languages > processing ${data.user.repositories.nodes.length} repositories`) @@ -59,17 +60,18 @@ export default async function({login, data, imports, q, rest, account}, {enabled //Indepth mode if (indepth) { console.debug(`metrics/compute/${login}/plugins > languages > switching to indepth mode (this may take some time)`) - Object.assign(languages, await indepth_analyzer({login, data, imports}, {skipped})) + Object.assign(languages, await indepth_analyzer({login, data, imports, repositories}, {skipped})) } //Compute languages stats - for (const {section, stats = {}, total = 0} of [{section:"favorites", stats:languages.stats, total:languages.total}, {section:"recent", ...languages["stats.recent"]}]) { + for (const {section, stats = {}, lines = {}, total = 0} of [{section:"favorites", stats:languages.stats, lines:languages.lines, total:languages.total}, {section:"recent", ...languages["stats.recent"]}]) { console.debug(`metrics/compute/${login}/plugins > languages > computing stats ${section}`) languages[section] = Object.entries(stats).filter(([name]) => !ignored.includes(name.toLocaleLowerCase())).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size:value, color:languages.colors[name], x:0})).filter(({value}) => value / total > threshold) const visible = {total:Object.values(languages[section]).map(({size}) => size).reduce((a, b) => a + b, 0)} for (let i = 0; i < languages[section].length; i++) { languages[section][i].value /= visible.total languages[section][i].x = (languages[section][i - 1]?.x ?? 0) + (languages[section][i - 1]?.value ?? 0) + languages[section][i].lines = lines[languages[section][i].name] ?? 0 if ((colors[i]) && (!colors[languages[section][i].name.toLocaleLowerCase()])) languages[section][i].color = colors[i] } diff --git a/source/plugins/languages/metadata.yml b/source/plugins/languages/metadata.yml index ae91b53e..d91f7d84 100644 --- a/source/plugins/languages/metadata.yml +++ b/source/plugins/languages/metadata.yml @@ -73,6 +73,7 @@ inputs: values: - bytes-size # Languages total size written in bytes - percentage # Languages proportions in % + - lines # Estimation of lines of code (plugin_languages_indepth must be enabled) default: "" example: bytes-size, percentage diff --git a/source/templates/classic/partials/languages.ejs b/source/templates/classic/partials/languages.ejs index 50bf52fc..68d38ce8 100644 --- a/source/templates/classic/partials/languages.ejs +++ b/source/templates/classic/partials/languages.ejs @@ -5,10 +5,10 @@ <%= plugins.languages.unique %> Language<%= s(plugins.languages.unique) %> - <% for (const section of plugins.languages.sections) { const languages = {"most-used":plugins.languages.favorites, "recently-used":plugins.languages.recent}[section] %> + <% for (const section of (plugins.languages.sections ?? ["error"])) { const languages = {"most-used":plugins.languages.favorites, "recently-used":plugins.languages.recent}[section] %>

- <%= {"most-used":"Most used languages", "recently-used":"Recently used languages"}[section] %> + <%= {"most-used":"Most used languages", "recently-used":"Recently used languages", error:""}[section] %>

<% if (plugins.languages.error) { %>
@@ -31,13 +31,14 @@
<% for (const row of rows) { %>
- <% for (const {name, value, color, size} of languages.filter((_, i) => i%rows.length === row)) { %> + <% for (const {name, value, lines, color, size} of languages.filter((_, i) => i%rows.length === row)) { %>
" fill-rule="evenodd" d="M8 4a4 4 0 100 8 4 4 0 000-8z"> <%= name %>
+ <% if (plugins.languages.details.includes("lines")) { %>
<%= f(lines) %> line<%= s(lines) %>
<% } %> <% if (plugins.languages.details.includes("bytes-size")) { %>
<%= f.bytes(size) %>
<% } %> <% if (plugins.languages.details.includes("percentage")) { %>
<%= f.percentage(value) %>
<% } %> diff --git a/source/templates/classic/style.css b/source/templates/classic/style.css index e0501161..affddea5 100644 --- a/source/templates/classic/style.css +++ b/source/templates/classic/style.css @@ -186,6 +186,9 @@ .field.language.details > *, .field.language.details small > * { flex: 1 1 0; } + .field.language.details small > *:not(:last-child) { + margin-right: 6px; + } /* Follow-up */ .followup.legend { diff --git a/source/templates/repository/partials/languages.ejs b/source/templates/repository/partials/languages.ejs index d5879b6e..68d38ce8 100644 --- a/source/templates/repository/partials/languages.ejs +++ b/source/templates/repository/partials/languages.ejs @@ -5,56 +5,59 @@ <%= plugins.languages.unique %> Language<%= s(plugins.languages.unique) %>
-
-

- Most used languages -

- <% if (plugins.languages.error) { %> -
-
- - <%= plugins.languages.error.message %> -
-
- <% } else { const width = 460 * (1 + large) %> - - - - - - <% for (const {name, value, color, x} of plugins.languages.favorites) { %> - - <% } %> - - <% if (plugins.languages.details?.length) { const rows = large ? [0, 1, 2, 3] : [0, 1] %> -
- <% for (const row of rows) { %> -
- <% for (const {name, value, color, size} of plugins.languages.favorites.filter((_, i) => i%rows.length === row)) { %> -
-
- - <%= name %> + <% for (const section of (plugins.languages.sections ?? ["error"])) { const languages = {"most-used":plugins.languages.favorites, "recently-used":plugins.languages.recent}[section] %> +
+

+ <%= {"most-used":"Most used languages", "recently-used":"Recently used languages", error:""}[section] %> +

+ <% if (plugins.languages.error) { %> +
+
+ + <%= plugins.languages.error.message %> +
+
+ <% } else { const width = 460 * (1 + large) %> + + + + + + <% for (const {name, value, color, x} of languages) { %> + "/> + <% } %> + + <% if (plugins.languages.details?.length) { const rows = large ? [0, 1, 2, 3] : [0, 1] %> +
+ <% for (const row of rows) { %> +
+ <% for (const {name, value, lines, color, size} of languages.filter((_, i) => i%rows.length === row)) { %> +
+
+ " fill-rule="evenodd" d="M8 4a4 4 0 100 8 4 4 0 000-8z"> + <%= name %> +
+ + <% if (plugins.languages.details.includes("lines")) { %>
<%= f(lines) %> line<%= s(lines) %>
<% } %> + <% if (plugins.languages.details.includes("bytes-size")) { %>
<%= f.bytes(size) %>
<% } %> + <% if (plugins.languages.details.includes("percentage")) { %>
<%= f.percentage(value) %>
<% } %> +
- - <% if (plugins.languages.details.includes("bytes-size")) { %>
<%= f.bytes(size) %>
<% } %> - <% if (plugins.languages.details.includes("percentage")) { %>
<%= f.percentage(value) %>
<% } %> - -
- <% } %> -
- <% } %> -
- <% } else { %> -
- <% for (const {name, value, color} of plugins.languages.favorites) { %> -
- - <%= name %> -
- <% } %> -
+ <% } %> +
+ <% } %> +
+ <% } else { %> +
+ <% for (const {name, value, color} of languages) { %> +
+ + <%= name %> +
+ <% } %> +
+ <% } %> <% } %> - <% } %> -
+
+ <% } %> <% } %> \ No newline at end of file