feat(plugins/languages): various improvements (#985)

source/plugins/languages/analyzers.mjs

@@ -1,3 +1,4 @@
+//Imports
 import linguist from "linguist-js"
 
 /**Indepth analyzer */
@@ -29,12 +30,12 @@ export async function indepth({login, data, imports, repositories, gpg}, {skippe
       finally {
         //Cleaning
         console.debug(`metrics/compute/${login}/plugins > languages > indepth > cleaning ${path}`)
-        await imports.fs.rm(path, {recursive:true, force:true})
+        await imports.fs.rm(path, {recursive:true, force:true}).catch(error => console.debug(`metrics/compute/${login}/plugins > languages > indepth > failed to clean ${path} (${error})`))
       }
     }
 
     //Compute repositories stats from fetched repositories
-    const results = {total:0, lines:{}, stats:{}, colors:{}, commits:0, files:0, missed:0, verified:{signature:0}}
+    const results = {total:0, lines:{}, stats:{}, colors:{}, commits:0, files:0, missed:{lines:0, bytes:0, commits:0}, verified:{signature:0}}
     for (const repository of repositories) {
       //Skip repository if asked
       if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) {
@@ -67,7 +68,7 @@ export async function indepth({login, data, imports, repositories, gpg}, {skippe
       finally {
         //Cleaning
         console.debug(`metrics/compute/${login}/plugins > languages > indepth > cleaning temp dir ${path}`)
-        await imports.fs.rm(path, {recursive:true, force:true})
+        await imports.fs.rm(path, {recursive:true, force:true}).catch(error => console.debug(`metrics/compute/${login}/plugins > languages > indepth > failed to clean ${path} (${error})`))
       }
     }
     solve(results)
@@ -85,7 +86,7 @@ export async function recent({login, data, imports, rest, account}, {skipped = [
 
     //Get user recent activity
     console.debug(`metrics/compute/${login}/plugins > languages > querying api`)
-    const commits = [], pages = Math.ceil(load / 100), results = {total:0, lines:{}, stats:{}, colors:{}, commits:0, files:0, missed:0, days}
+    const commits = [], pages = Math.ceil(load / 100), results = {total:0, lines:{}, stats:{}, colors:{}, commits:0, files:0, missed:{lines:0, bytes:0, commits:0}, days}
     try {
       for (let page = 1; page <= pages; page++) {
         console.debug(`metrics/compute/${login}/plugins > languages > loading page ${page}`)
@@ -134,7 +135,7 @@ export async function recent({login, data, imports, rest, account}, {skipped = [
       await imports.fs.mkdir(path, {recursive:true})
       await Promise.all(patches.map(async ({name, directory, patch}) => {
         await imports.fs.mkdir(imports.paths.join(path, directory), {recursive:true})
-        imports.fs.writeFile(imports.paths.join(path, directory, name), patch)
+        await imports.fs.writeFile(imports.paths.join(path, directory, name), patch)
       }))
 
       //Process temporary repositories
@@ -170,7 +171,7 @@ export async function recent({login, data, imports, rest, account}, {skipped = [
     finally {
       //Cleaning
       console.debug(`metrics/compute/${login}/plugins > languages > cleaning temp dir ${path}`)
-      await imports.fs.rm(path, {recursive:true, force:true})
+      await imports.fs.rm(path, {recursive:true, force:true}).catch(error => console.debug(`metrics/compute/${login}/plugins > languages > indepth > failed to clean ${path} (${error})`))
     }
     solve(results)
   })
@@ -199,7 +200,7 @@ async function analyze({login, imports, data}, {results, path, categories = ["pr
     try {
       console.debug(`metrics/compute/${login}/plugins > languages > indepth > processing commits ${page * per_page} from ${(page + 1) * per_page}`)
       let empty = true, file = null, lang = null
-      await imports.spawn("git", ["log", ...data.shared["commits.authoring"].map(authoring => `--author="${authoring}"`), "--regexp-ignore-case", "--format=short", "--patch", `--max-count=${per_page}`, `--skip=${page * per_page}`], {cwd:path}, {
+      await imports.spawn("git", ["log", ...data.shared["commits.authoring"].map(authoring => `--author="${authoring}"`), "--regexp-ignore-case", "--format=short", "--no-merges", "--patch", `--max-count=${per_page}`, `--skip=${page * per_page}`], {cwd:path}, {
         stdout(line) {
           try {
             //Unflag empty output
@@ -226,8 +227,8 @@ async function analyze({login, imports, data}, {results, path, categories = ["pr
             //File marker
             if (/^[+]{3}\sb[/](?<file>[\s\S]+)$/.test(line)) {
               file = `${path}/${line.match(/^[+]{3}\sb[/](?<file>[\s\S]+)$/)?.groups?.file}`.replace(/\\/g, "/")
-              lang = files[file] ?? null
+              lang = files[file] ?? "<unknown>"
-              if ((lang) && (!categories.includes(languageResults[lang].type)))
+              if ((lang) && (lang !== "<unknown>") && (!categories.includes(languageResults[lang].type)))
                 lang = null
               edited.add(file)
               return
@@ -238,9 +239,15 @@ async function analyze({login, imports, data}, {results, path, categories = ["pr
             //Added line marker
             if (/^[+]\s*(?<line>[\s\S]+)$/.test(line)) {
               const size = Buffer.byteLength(line.match(/^[+]\s*(?<line>[\s\S]+)$/)?.groups?.line ?? "", "utf-8")
+              results.total += size
+              if (lang === "<unknown>") {
+                results.missed.lines++
+                results.missed.bytes += size
+              }
+              else {
                 results.stats[lang] = (results.stats[lang] ?? 0) + size
                 results.lines[lang] = (results.lines[lang] ?? 0) + 1
-              results.total += size
+              }
             }
           }
           catch (error) {
@@ -255,7 +262,7 @@ async function analyze({login, imports, data}, {results, path, categories = ["pr
     }
     catch {
       console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured on page ${page}, skipping...`)
-      results.missed += per_page
+      results.missed.commits += per_page
     }
   }
   await Promise.allSettled(pending)
@@ -278,7 +285,7 @@ if (/languages.analyzers.mjs$/.test(process.argv[1])) {
 
     //Prepare call
     const imports = await import("../../app/metrics/utils.mjs")
-    const results = {total:0, lines:{}, colors:{}, stats:{}, missed:0}
+    const results = {total:0, lines:{}, colors:{}, stats:{}, missed:{lines:0, bytes:0, commits:0}}
     console.debug = log => /exited with code null/.test(log) ? console.error(log.replace(/^.*--max-count=(?<step>\d+) --skip=(?<start>\d+).*$/, (_, step, start) => `error: skipped commits ${start} from ${Number(start) + Number(step)}`)) : null
 
     //Analyze repository

source/plugins/languages/index.mjs

@@ -12,12 +12,12 @@ export default async function({login, data, imports, q, rest, account}, {enabled
     //Context
     let context = {mode:"user"}
     if (q.repo) {
-      console.debug(`metrics/compute/${login}/plugins > activity > switched to repository mode`)
+      console.debug(`metrics/compute/${login}/plugins > languages > switched to repository mode`)
       context = {...context, mode:"repository"}
     }
 
     //Load inputs
-    let {ignored, skipped, colors, aliases, details, threshold, limit, indepth, "analysis.timeout":timeout, sections, categories, "recent.categories":_recent_categories, "recent.load":_recent_load, "recent.days":_recent_days} = imports.metadata.plugins.languages.inputs({
+    let {ignored, skipped, other, colors, aliases, details, threshold, limit, indepth, "analysis.timeout":timeout, sections, categories, "recent.categories":_recent_categories, "recent.load":_recent_load, "recent.days":_recent_days} = imports.metadata.plugins.languages.inputs({
       data,
       account,
       q,
@@ -102,7 +102,7 @@ export default async function({login, data, imports, q, rest, account}, {enabled
           const existingColors = languages.colors
           Object.assign(languages, await indepth_analyzer({login, data, imports, repositories, gpg}, {skipped, categories, timeout}))
           Object.assign(languages.colors, existingColors)
-          console.debug(`metrics/compute/${login}/plugins > languages > indepth analysis missed ${languages.missed} commits`)
+          console.debug(`metrics/compute/${login}/plugins > languages > indepth analysis missed ${languages.missed.commits} commits`)
         }
         catch (error) {
           console.debug(`metrics/compute/${login}/plugins > languages > ${error}`)
@@ -125,10 +125,20 @@ export default async function({login, data, imports, q, rest, account}, {enabled
     }
 
     //Compute languages stats
-    for (const {section, stats = {}, lines = {}, total = 0} of [{section:"favorites", stats:languages.stats, lines:languages.lines, total:languages.total}, {section:"recent", ...languages["stats.recent"]}]) {
+    for (const {section, stats = {}, lines = {}, missed = {bytes:0}, total = 0} of [{section:"favorites", stats:languages.stats, lines:languages.lines, total:languages.total, missed:languages.missed}, {section:"recent", ...languages["stats.recent"]}]) {
       console.debug(`metrics/compute/${login}/plugins > languages > computing stats ${section}`)
-      languages[section] = Object.entries(stats).filter(([name]) => !ignored.includes(name.toLocaleLowerCase())).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size:value, color:languages.colors[name], x:0})).filter(({value}) => value / total > threshold
+      languages[section] = Object.entries(stats).filter(([name]) => !ignored.includes(name.toLocaleLowerCase())).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size:value, color:languages.colors[name], x:0})).filter(({value}) => value / total > threshold)
-      )
+      if (other) {
+        let value = indepth ? missed.bytes : Object.entries(stats).filter(([name]) => !Object.values(languages[section]).map(({name}) => name).includes(name)).reduce((a, [_, b]) => a + b, 0)
+        if (value) {
+          if (languages[section].length === limit) {
+            const {size} = languages[section].pop()
+            value += size
+          }
+          //dprint-ignore-next-line
+          languages[section].push({name:"Other", value, size:value, get lines() { return missed.lines }, set lines(_) { }, x:0}) //eslint-disable-line brace-style, no-empty-function, max-statements-per-line
+        }
+      }
       const visible = {total:Object.values(languages[section]).map(({size}) => size).reduce((a, b) => a + b, 0)}
       for (let i = 0; i < languages[section].length; i++) {
         const {name} = languages[section][i]

source/plugins/languages/metadata.yml

@@ -48,6 +48,15 @@ inputs:
     type: string
     default: 0%
 
+  plugin_languages_other:
+    description: |
+      Group unknown, ignored and over-limit languages into a single "Other" category
+
+      If this option is enabled, "Other" category will not be subject to `plugin_languages_threshold`.
+      It will be automatically hidden if empty.
+    type: boolean
+    default: no
+
   plugin_languages_colors:
     description: Custom languages colors
     type: array

source/templates/classic/partials/languages.ejs

@@ -20,13 +20,22 @@
       <% } else { const width = 460 * (1 + large) %>
         <% if (section === "recently-used") { %>
           <small>
-            estimation from <%= plugins.languages["stats.recent"]?.files %> edited file<%= s(plugins.languages["stats.recent"]?.files) %> from <%= plugins.languages["stats.recent"]?.commits %> commit<%= s(plugins.languages["stats.recent"]?.commits) %> over last <%= plugins.languages["stats.recent"]?.latest ?? plugins.languages["stats.recent"]?.days %> day<%= s(plugins.languages["stats.recent"]?.latest ?? plugins.languages["stats.recent"]?.days) %>
+            <% if (languages.length) { %>
+              estimation from <%= f(plugins.languages["stats.recent"]?.total) %>b of code in <%= plugins.languages["stats.recent"]?.files %> edited file<%= s(plugins.languages["stats.recent"]?.files) %> across <%= plugins.languages["stats.recent"]?.commits %> commit<%= s(plugins.languages["stats.recent"]?.commits) %> over last <%= plugins.languages["stats.recent"]?.latest ?? plugins.languages["stats.recent"]?.days %> day<%= s(plugins.languages["stats.recent"]?.latest ?? plugins.languages["stats.recent"]?.days) %>
+            <% } else { %>
+              No recent push activity found over last <%= plugins.languages["stats.recent"]?.latest ?? plugins.languages["stats.recent"]?.days %> day<%= s(plugins.languages["stats.recent"]?.latest ?? plugins.languages["stats.recent"]?.days) %>
+            <% } %>
           </small>
         <% } else if ((section === "most-used")&&(plugins.languages.indepth)) { %>
           <small>
-            estimation from <%= plugins.languages.files %> edited file<%= s(plugins.languages.files) %> from <%= plugins.languages.commits %> commit<%= s(plugins.languages.commits) %>
+            <% if (languages.length) { %>
+              estimation from <%= f(plugins.languages.total) %>b of code in <%= plugins.languages.files %> edited file<%= s(plugins.languages.files) %> across <%= plugins.languages.commits %> commit<%= s(plugins.languages.commits) %>
+            <% } else { %>
+              No push activity found
+            <% } %>
           </small>
         <% } %>
+        <% if (languages.length) { %>
           <svg class="bar" xmlns="http://www.w3.org/2000/svg" width="<%= width %>" height="8">
             <mask id="languages-bar">
               <rect x="0" y="0" width="<%= width %>" height="8" fill="white" rx="5"/>
@@ -36,6 +45,7 @@
               <rect mask="url(#languages-bar)" x="<%= x*width %>" y="0" width="<%= value*width %>" height="8" fill="<%= color ?? "#959DA5" %>"/>
             <% } %>
           </svg>
+        <% } %>
         <% if (plugins.languages.details.length) { const rows = large ? [0, 1, 2, 3] : (plugins.languages.details.length > 2) ? [0] : [0, 1] %>
           <div class="row fill-width">
             <% for (const row of rows) { %>