Add indepth languages analysis (#325)

source/app/metrics/utils.mjs

@@ -131,7 +131,7 @@ export async function chartist() {
 }
 
 /**Run command */
-export async function run(command, options, {prefixed = true} = {}) {
+export async function run(command, options, {prefixed = true, log = true} = {}) {
   const prefix = {win32:"wsl"}[process.platform] ?? ""
   command = `${prefixed ? prefix : ""} ${command}`.trim()
   return new Promise((solve, reject) => {
@@ -142,8 +142,10 @@ export async function run(command, options, {prefixed = true} = {}) {
     child.stderr.on("data", data => stderr += data)
     child.on("close", code => {
       console.debug(`metrics/command > ${command} > exited with code ${code}`)
-      console.debug(stdout)
-      console.debug(stderr)
+      if (log) {
+        console.debug(stdout)
+        console.debug(stderr)
+      }
       return code === 0 ? solve(stdout) : reject(stderr)
     })
   })

source/plugins/languages/README.md

@@ -17,6 +17,16 @@ You can specify either an index with a color, or a language name (case insensiti
 Colors can be either in hexadecimal format or a [named color](https://developer.mozilla.org/en-US/docs/Web/CSS/color_value).
 It is also possible to use a predefined set of colors from [colorsets.json](colorsets.json)
 
+**Using `indepth` statistics**
+
+Languages statistics are computed using the top languages of each repository you contributed to.
+If you work a lot with other people, these numbers may be less representative of your actual work.
+
+The `plugin_languages_indepth` option lets you get more accurate metrics by cloning each repository, running [github/linguist](https://github.com/github/linguist) on it and iterating over patches matching your username from `git log`, but will be **significantly slower**.
+
+> ⚠️ Although *metrics* does not send any code to external sources, you must understand that when using this option repositories are cloned locally temporarly on the GitHub Action runner. If you work with sensitive data or company code, it is advised to keep this option disabled. *Metrics* cannot be held responsible for any eventual code leaks, use at your own risk.
+> Source code is available for auditing at [indepth.mjs](/source/plugins/languages/indepth.mjs)
+
 #### ℹ️ Examples workflows
 
 [➡️ Available options for this plugin](metadata.yml)
@@ -32,4 +42,5 @@ It is also possible to use a predefined set of colors from [colorsets.json](colo
     plugin_languages_details: bytes-size, percentage             # Additionally display total bytes size and percentage
     plugin_languages_threshold: 2%                               # Hides all languages less than 2%
     plugin_languages_limit: 8                                    # Display up to 8 languages
+    plugin_languages_indepth: no                                 # Get indepth stats (see documentation before enabling)
 ```

source/plugins/languages/indepth.mjs

@@ -0,0 +1,105 @@
+/**Indepth analyzer */
+export default async function({login, data, imports}, {skipped, ignored}) {
+  //Check prerequisites
+  if (!await imports.which("github-linguist"))
+    throw new Error("Feature requires github-linguist")
+
+  //Compute repositories stats
+  const results = {total:0, stats:{}}
+  for (const repository of data.user.repositories.nodes) {
+    const repo = `${repository.owner.login}/${repository.name}`
+    console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking ${repo}`)
+    //Skip repository if asked
+    if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) {
+      console.debug(`metrics/compute/${login}/plugins > languages > skipped repository ${repository.owner.login}/${repository.name}`)
+      continue
+    }
+    //Analyze
+    try {
+      await analyze(arguments[0], {repo, results})
+    }
+    catch {
+      console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured while processing ${repo}, skipping...`)
+    }
+  }
+
+  //Ignore languages if asked
+  Object.assign(results.stats, Object.fromEntries(Object.entries(results.stats).filter(([lang]) => !ignored.includes(lang.toLocaleLowerCase()))))
+  return results
+}
+
+/**Clone and analyze a single repository */
+async function analyze({login, data, imports}, {repo, results}) {
+  //Git clone into a temporary directory
+  const path = imports.paths.join(imports.os.tmpdir(), `${data.user.databaseId}-${repo.replace(/[^\w]/g, "_")}`)
+  console.debug(`metrics/compute/${login}/plugins > languages > indepth > cloning ${repo} to temp dir ${path}`)
+  await imports.fs.rmdir(path, {recursive:true})
+  await imports.fs.mkdir(path, {recursive:true})
+  const git = await imports.git(path)
+  await git.clone(`https://github.com/${repo}`, ".").status()
+
+  //Spawn linguist process and map files to languages
+  console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`)
+  const files = {}
+  {
+    const stdout = await imports.run("github-linguist --breakdown", {cwd:path}, {log:false})
+    let lang = null
+    for (const line of stdout.split("\n").map(line => line.trim())) {
+      //Ignore empty lines
+      if (!line.length)
+        continue
+      //Language marker
+      if (/^(?<lang>[\s\S]+):\s*$/.test(line)) {
+        lang = line.match(/^(?<lang>[\s\S]+):\s*$/)?.groups?.lang ?? null
+        continue
+      }
+      //Store language
+      if (lang) {
+        files[line] = {lang}
+        continue
+      }
+    }
+  }
+
+  //Processing diff
+  const per_page = 10
+  console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking git log`)
+  for (let page = 0; ; page++) {
+    try {
+      const stdout = await imports.run(`git log --author="${login}" --format="" --patch --max-count=${per_page} --skip=${page*per_page}`, {cwd:path}, {log:false})
+      let file = null, lang = null
+      if (!stdout.trim().length) {
+        console.debug(`metrics/compute/${login}/plugins > languages > indepth > no more commits`)
+        break
+      }
+      console.debug(`metrics/compute/${login}/plugins > languages > indepth > processing commits ${page*per_page} from ${(page+1)*per_page}`)
+      for (const line of stdout.split("\n").map(line => line.trim())) {
+        //Ignore empty lines or unneeded lines
+        if ((!/^[+]/.test(line))||(!line.length))
+          continue
+        //File marker
+        if (/^[+]{3}\sb[/](?<file>[\s\S]+)$/.test(line)) {
+          file = line.match(/^[+]{3}\sb[/](?<file>[\s\S]+)$/)?.groups?.file ?? null
+          lang = files[file]?.lang ?? null
+          continue
+        }
+        //Ignore unkonwn languages
+        if (!lang)
+          continue
+        //Added line marker
+        if (/^[+]\s(?<line>[\s\S]+)$/.test(line)) {
+          const size = Buffer.byteLength(line.match(/^[+]\s(?<line>[\s\S]+)$/)?.groups?.line ?? "", "utf-8")
+          results.stats[lang] = (results.stats[lang] ?? 0) + size
+          results.total += size
+        }
+      }
+    }
+    catch {
+      console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured on page ${page}, skipping...`)
+    }
+  }
+
+  //Cleaning
+  console.debug(`metrics/compute/${login}/plugins > languages > indepth > cleaning temp dir ${path}`)
+  await imports.fs.rmdir(path, {recursive:true})
+}

source/plugins/languages/index.mjs

@@ -1,3 +1,6 @@
+//Imports
+import indepth_analyzer from "./indepth.mjs"
+
 //Setup
 export default async function({login, data, imports, q, account}, {enabled = false} = {}) {
   //Plugin execution
@@ -7,7 +10,7 @@ export default async function({login, data, imports, q, account}, {enabled = fal
       return null
 
     //Load inputs
-    let {ignored, skipped, colors, details, threshold, limit} = imports.metadata.plugins.languages.inputs({data, account, q})
+    let {ignored, skipped, colors, details, threshold, limit, indepth} = imports.metadata.plugins.languages.inputs({data, account, q})
     threshold = (Number(threshold.replace(/%$/, "")) || 0) / 100
     skipped.push(...data.shared["repositories.skipped"])
     if (!limit)
@@ -43,6 +46,12 @@ export default async function({login, data, imports, q, account}, {enabled = fal
       }
     }
 
+    //Indepth mode
+    if (indepth) {
+      console.debug(`metrics/compute/${login}/plugins > languages > switching to indepth mode (this may take some time)`)
+      Object.assign(languages, await indepth_analyzer({login, data, imports}, {skipped, ignored}))
+    }
+
     //Compute languages stats
     console.debug(`metrics/compute/${login}/plugins > languages > computing stats`)
     languages.favorites = Object.entries(languages.stats).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size:value, color:languages.colors[name], x:0})).filter(({value}) => value / languages.total > threshold)

source/plugins/languages/metadata.yml

@@ -70,3 +70,10 @@ inputs:
     description: Minimum threshold
     type: string
     default: 0%
+
+  # Compute indepth languages statistics by cloning repositories and processing your commits individually
+  # See documentation before enabling
+  plugin_languages_indepth:
+    description: Indepth languages processing (see documentation before enabling)
+    type: boolean
+    default: false