Improvements languages indepth analysis (#329)

source/app/mocks/api/github/graphql/base.repositories.mjs

@@ -4,6 +4,9 @@ export default function({faker, query, login = faker.internet.userName()}) {
   return /after: "MOCKED_CURSOR"/m.test(query)
     ? ({
       user:{
+        get repositoriesContributedTo() {
+          return this.repositories
+        },
         repositories:{
           edges:[],
           nodes:[],
@@ -12,6 +15,9 @@ export default function({faker, query, login = faker.internet.userName()}) {
     })
     : ({
       user:{
+        get repositoriesContributedTo() {
+          return this.repositories
+        },
         repositories:{
           edges:[
             {

source/plugins/base/index.mjs

@@ -34,22 +34,24 @@ export default async function({login, graphql, data, q, queries, imports}, conf)
       Object.assign(data, {user:queried[account]})
       postprocess?.[account]({login, data})
       //Query repositories from GitHub API
-      {
+      data.user.repositoriesContributedTo.nodes = data.user.repositoriesContributedTo.nodes ?? []
+      for (const type of ["repositories", "repositoriesContributedTo"]) {
         //Iterate through repositories
         let cursor = null
         let pushed = 0
+        const options = {repositories:{forks, affiliations, constraints:""}, repositoriesContributedTo:{forks:"", affiliations:"", constraints:", includeUserRepositories: false, contributionTypes: COMMIT"}}[type] ?? null
         do {
-          console.debug(`metrics/compute/${login}/base > retrieving repositories after ${cursor}`)
+          console.debug(`metrics/compute/${login}/base > retrieving ${type} after ${cursor}`)
-          const {[account]:{repositories:{edges, nodes}}} = await graphql(queries.base.repositories({login, account, after:cursor ? `after: "${cursor}"` : "", repositories:Math.min(repositories, {user:100, organization:25}[account]), forks, affiliations}))
+          const {[account]:{[type]:{edges = [], nodes = []} = {}}} = await graphql(queries.base.repositories({login, account, type, after:cursor ? `after: "${cursor}"` : "", repositories:Math.min(repositories, {user:100, organization:25}[account]), ...options}))
           cursor = edges?.[edges?.length - 1]?.cursor
-          data.user.repositories.nodes.push(...nodes)
+          data.user[type].nodes.push(...nodes)
           pushed = nodes.length
-          console.debug(`metrics/compute/${login}/base > retrieved ${pushed} repositories after ${cursor}`)
+          console.debug(`metrics/compute/${login}/base > retrieved ${pushed} ${type} after ${cursor}`)
-        } while ((pushed) && (cursor) && (data.user.repositories.nodes.length < repositories))
+        } while ((pushed) && (cursor) && (data.user.repositories.nodes.length + data.user.repositoriesContributedTo.nodes.length < repositories))
         //Limit repositories
-        console.debug(`metrics/compute/${login}/base > keeping only ${repositories} repositories`)
+        console.debug(`metrics/compute/${login}/base > keeping only ${repositories} ${type}`)
-        data.user.repositories.nodes.splice(repositories)
+        data.user[type].nodes.splice(repositories)
-        console.debug(`metrics/compute/${login}/base > loaded ${data.user.repositories.nodes.length} repositories`)
+        console.debug(`metrics/compute/${login}/base > loaded ${data.user[type].nodes.length} ${type}`)
       }
       //Success
       console.debug(`metrics/compute/${login}/base > graphql query > account ${account} > success`)

source/plugins/base/queries/repositories.graphql

@@ -1,6 +1,6 @@
 query BaseRepositories {
   $account(login: "$login") {
-    repositories($after first: $repositories $forks $affiliations, orderBy: {field: UPDATED_AT, direction: DESC}) {
+    $type($after first: $repositories $forks $affiliations $constraints, orderBy: {field: UPDATED_AT, direction: DESC}) {
       edges {
         cursor
       }

source/plugins/languages/analyzers.mjs

@@ -1,12 +1,12 @@
 /**Indepth analyzer */
-export async function indepth({login, data, imports}, {skipped}) {
+export async function indepth({login, data, imports, repositories}, {skipped}) {
   //Check prerequisites
   if (!await imports.which("github-linguist"))
     throw new Error("Feature requires github-linguist")
 
   //Compute repositories stats from fetched repositories
-  const results = {total:0, stats:{}}
+  const results = {total:0, lines:{}, stats:{}}
-  for (const repository of data.user.repositories.nodes) {
+  for (const repository of repositories) {
     //Skip repository if asked
     if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) {
       console.debug(`metrics/compute/${login}/plugins > languages > skipped repository ${repository.owner.login}/${repository.name}`)
@@ -52,7 +52,7 @@ export async function recent({login, data, imports, rest, account}, {skipped}) {
 
   //Get user recent activity
   console.debug(`metrics/compute/${login}/plugins > languages > querying api`)
-  const commits = [], days = 14, pages = 3, results = {total:0, stats:{}}
+  const commits = [], days = 14, pages = 3, results = {total:0, lines:{}, stats:{}}
   try {
     for (let page = 1; page <= pages; page++) {
       console.debug(`metrics/compute/${login}/plugins > languages > loading page ${page}`)
@@ -110,8 +110,6 @@ export async function recent({login, data, imports, rest, account}, {skipped}) {
     console.debug(`metrics/compute/${login}/plugins > languages > cleaning temp dir ${path}`)
     await imports.fs.rmdir(path, {recursive:true})
   }
-
-  console.log(results)
   return results
 }
 
@@ -121,8 +119,6 @@ async function analyze({login, imports}, {results, path}) {
   console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`)
   const files = Object.fromEntries(Object.entries(JSON.parse(await imports.run("github-linguist --json", {cwd:path}, {log:false}))).flatMap(([lang, files]) => files.map(file => [file, lang])))
 
-  console.log(files)
-
   //Processing diff
   const per_page = 10
   console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking git log`)
@@ -149,9 +145,10 @@ async function analyze({login, imports}, {results, path}) {
         if (!lang)
           continue
         //Added line marker
-        if (/^[+]\s(?<line>[\s\S]+)$/.test(line)) {
+        if (/^[+]\s*(?<line>[\s\S]+)$/.test(line)) {
-          const size = Buffer.byteLength(line.match(/^[+]\s(?<line>[\s\S]+)$/)?.groups?.line ?? "", "utf-8")
+          const size = Buffer.byteLength(line.match(/^[+]\s*(?<line>[\s\S]+)$/)?.groups?.line ?? "", "utf-8")
           results.stats[lang] = (results.stats[lang] ?? 0) + size
+          results.lines[lang] = (results.lines[lang] ?? 0) + 1
           results.total += size
         }
       }

source/plugins/languages/index.mjs

@@ -31,7 +31,8 @@ export default async function({login, data, imports, q, rest, account}, {enabled
     console.debug(`metrics/compute/${login}/plugins > languages > custom colors ${JSON.stringify(colors)}`)
 
     //Unique languages
-    const unique = new Set(data.user.repositories.nodes.flatMap(repository => repository.languages.edges.map(({node:{name}}) => name))).size
+    const repositories = [...data.user.repositories.nodes, ...data.user.repositoriesContributedTo.nodes]
+    const unique = new Set(repositories.flatMap(repository => repository.languages.edges.map(({node:{name}}) => name))).size
 
     //Iterate through user's repositories and retrieve languages data
     console.debug(`metrics/compute/${login}/plugins > languages > processing ${data.user.repositories.nodes.length} repositories`)
@@ -59,17 +60,18 @@ export default async function({login, data, imports, q, rest, account}, {enabled
     //Indepth mode
     if (indepth) {
       console.debug(`metrics/compute/${login}/plugins > languages > switching to indepth mode (this may take some time)`)
-      Object.assign(languages, await indepth_analyzer({login, data, imports}, {skipped}))
+      Object.assign(languages, await indepth_analyzer({login, data, imports, repositories}, {skipped}))
     }
 
     //Compute languages stats
-    for (const {section, stats = {}, total = 0} of [{section:"favorites", stats:languages.stats, total:languages.total}, {section:"recent", ...languages["stats.recent"]}]) {
+    for (const {section, stats = {}, lines = {}, total = 0} of [{section:"favorites", stats:languages.stats, lines:languages.lines, total:languages.total}, {section:"recent", ...languages["stats.recent"]}]) {
       console.debug(`metrics/compute/${login}/plugins > languages > computing stats ${section}`)
       languages[section] = Object.entries(stats).filter(([name]) => !ignored.includes(name.toLocaleLowerCase())).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size:value, color:languages.colors[name], x:0})).filter(({value}) => value / total > threshold)
       const visible = {total:Object.values(languages[section]).map(({size}) => size).reduce((a, b) => a + b, 0)}
       for (let i = 0; i < languages[section].length; i++) {
         languages[section][i].value /= visible.total
         languages[section][i].x = (languages[section][i - 1]?.x ?? 0) + (languages[section][i - 1]?.value ?? 0)
+        languages[section][i].lines = lines[languages[section][i].name] ?? 0
         if ((colors[i]) && (!colors[languages[section][i].name.toLocaleLowerCase()]))
           languages[section][i].color = colors[i]
       }

source/plugins/languages/metadata.yml

@@ -73,6 +73,7 @@ inputs:
     values:
       - bytes-size # Languages total size written in bytes
       - percentage # Languages proportions in %
+      - lines      # Estimation of lines of code (plugin_languages_indepth must be enabled)
     default: ""
     example: bytes-size, percentage
 

source/templates/classic/partials/languages.ejs

@@ -5,10 +5,10 @@
       <%= plugins.languages.unique %> Language<%= s(plugins.languages.unique) %>
     </h2>
   </section>
-  <% for (const section of plugins.languages.sections) { const languages = {"most-used":plugins.languages.favorites, "recently-used":plugins.languages.recent}[section] %>
+  <% for (const section of (plugins.languages.sections ?? ["error"])) { const languages = {"most-used":plugins.languages.favorites, "recently-used":plugins.languages.recent}[section] %>
     <section class="column">
       <h3 class="field">
-        <%= {"most-used":"Most used languages", "recently-used":"Recently used languages"}[section] %>
+        <%= {"most-used":"Most used languages", "recently-used":"Recently used languages", error:""}[section] %>
       </h3>
       <% if (plugins.languages.error) { %>
         <section>
@@ -31,13 +31,14 @@
           <div class="row fill-width">
             <% for (const row of rows) { %>
               <section>
-                <% for (const {name, value, color, size} of languages.filter((_, i) => i%rows.length === row)) { %>
+                <% for (const {name, value, lines, color, size} of languages.filter((_, i) => i%rows.length === row)) { %>
                   <div class="field language details">
                     <div class="field">
                       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path fill="<%= color ?? "#959DA5" %>" fill-rule="evenodd" d="M8 4a4 4 0 100 8 4 4 0 000-8z"></path></svg>
                       <%= name %>
                     </div>
                     <small>
+                      <% if (plugins.languages.details.includes("lines")) { %> <div><%= f(lines) %> line<%= s(lines) %></div><% } %>
                       <% if (plugins.languages.details.includes("bytes-size")) { %> <div><%= f.bytes(size) %></div><% } %>
                       <% if (plugins.languages.details.includes("percentage")) { %> <div><%= f.percentage(value) %></div><% } %>
                     </small>

source/templates/classic/style.css

@@ -186,6 +186,9 @@
   .field.language.details > *, .field.language.details small > * {
     flex: 1 1 0;
   }
+  .field.language.details small > *:not(:last-child) {
+    margin-right: 6px;
+  }
 
 /* Follow-up */
   .followup.legend {

source/templates/repository/partials/languages.ejs

@@ -5,9 +5,10 @@
       <%= plugins.languages.unique %> Language<%= s(plugins.languages.unique) %>
     </h2>
   </section>
+  <% for (const section of (plugins.languages.sections ?? ["error"])) { const languages = {"most-used":plugins.languages.favorites, "recently-used":plugins.languages.recent}[section] %>
     <section class="column">
       <h3 class="field">
-      Most used languages
+        <%= {"most-used":"Most used languages", "recently-used":"Recently used languages", error:""}[section] %>
       </h3>
       <% if (plugins.languages.error) { %>
         <section>
@@ -21,22 +22,23 @@
           <mask id="languages-bar">
             <rect x="0" y="0" width="<%= width %>" height="8" fill="white" rx="5"/>
           </mask>
-        <rect mask="url(#languages-bar)" x="0" y="0" width="<%= plugins.languages.favorites.length ? 0 : width %>" height="8" fill="#d1d5da"/>
+          <rect mask="url(#languages-bar)" x="0" y="0" width="<%= languages.length ? 0 : width %>" height="8" fill="#d1d5da"/>
-        <% for (const {name, value, color, x} of plugins.languages.favorites) { %>
+          <% for (const {name, value, color, x} of languages) { %>
-          <rect mask="url(#languages-bar)" x="<%= x*width %>" y="0" width="<%= value*width %>" height="8" fill="<%= color %>"/>
+            <rect mask="url(#languages-bar)" x="<%= x*width %>" y="0" width="<%= value*width %>" height="8" fill="<%= color ?? "#959DA5" %>"/>
           <% } %>
         </svg>
         <% if (plugins.languages.details?.length) { const rows = large ? [0, 1, 2, 3] : [0, 1] %>
           <div class="row fill-width">
             <% for (const row of rows) { %>
               <section>
-              <% for (const {name, value, color, size} of plugins.languages.favorites.filter((_, i) => i%rows.length === row)) { %>
+                <% for (const {name, value, lines, color, size} of languages.filter((_, i) => i%rows.length === row)) { %>
                   <div class="field language details">
                     <div class="field">
-                    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path fill="<%= color %>" fill-rule="evenodd" d="M8 4a4 4 0 100 8 4 4 0 000-8z"></path></svg>
+                      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path fill="<%= color ?? "#959DA5" %>" fill-rule="evenodd" d="M8 4a4 4 0 100 8 4 4 0 000-8z"></path></svg>
                       <%= name %>
                     </div>
                     <small>
+                      <% if (plugins.languages.details.includes("lines")) { %> <div><%= f(lines) %> line<%= s(lines) %></div><% } %>
                       <% if (plugins.languages.details.includes("bytes-size")) { %> <div><%= f.bytes(size) %></div><% } %>
                       <% if (plugins.languages.details.includes("percentage")) { %> <div><%= f.percentage(value) %></div><% } %>
                     </small>
@@ -47,7 +49,7 @@
           </div>
         <% } else { %>
           <div class="field center horizontal-wrap fill-width">
-          <% for (const {name, value, color} of plugins.languages.favorites) { %>
+            <% for (const {name, value, color} of languages) { %>
               <div class="field center no-wrap language">
                 <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path fill="<%= color %>" fill-rule="evenodd" d="M8 4a4 4 0 100 8 4 4 0 000-8z"></path></svg>
                 <%= name %>
@@ -58,3 +60,4 @@
       <% } %>
     </section>
   <% } %>
+<% } %>