Use spawn instead of exec to handle large git log outputs

source/app/metrics/utils.mjs

@@ -21,6 +21,7 @@ import twemojis from "twemoji-parser"
 import url from "url"
 import util from "util"
 import fetch from "node-fetch"
+import readline from "readline"
 prism_lang()
 
 //Exports
@@ -140,18 +141,18 @@ export async function chartist() {
     .replace(/class="ct-chart-line">/, `class="ct-chart-line">${css}`)
 }
 
-/**Run command */
+/**Run command (use this to execute commands and process whole output at once, may not be suitable for large outputs) */
 export async function run(command, options, {prefixed = true, log = true} = {}) {
   const prefix = {win32:"wsl"}[process.platform] ?? ""
   command = `${prefixed ? prefix : ""} ${command}`.trim()
   return new Promise((solve, reject) => {
-    console.debug(`metrics/command > ${command}`)
+    console.debug(`metrics/command/run > ${command}`)
     const child = processes.exec(command, options)
     let [stdout, stderr] = ["", ""]
     child.stdout.on("data", data => stdout += data)
     child.stderr.on("data", data => stderr += data)
     child.on("close", code => {
-      console.debug(`metrics/command > ${command} > exited with code ${code}`)
+      console.debug(`metrics/command/run > ${command} > exited with code ${code}`)
       if (log) {
         console.debug(stdout)
         console.debug(stderr)
@@ -161,6 +162,30 @@ export async function run(command, options, {prefixed = true, log = true} = {})
   })
 }
 
+/**Spawn command (use this to execute commands and process output on the fly) */
+export async function spawn(command, args = [], options = {}, {prefixed = true, timeout = 300*1000, stdout} = {}) {
+  const prefix = {win32:"wsl"}[process.platform] ?? ""
+  if ((prefixed)&&(prefix)) {
+    args.unshift(command)
+    command = prefix
+  }
+  if (!stdout)
+    throw new Error("`stdout` argument was not provided, use run() instead of spawn() if processing output is not needed")
+  return new Promise((solve, reject) => {
+    console.debug(`metrics/command/spawn > ${command} with ${args.join(" ")}`)
+    const child = processes.spawn(command, args, {...options, shell:true, timeout})
+    const reader = readline.createInterface({input:child.stdout})
+    reader.on("line", stdout)
+    const closed = new Promise(close => reader.on("close", close))
+    child.on("close", async code => {
+      console.debug(`metrics/command/spawn > ${command} with ${args.join(" ")} > exited with code ${code}`)
+      await closed
+      console.debug(`metrics/command/spawn > ${command} with ${args.join(" ")} > reader closed`)
+      return code === 0 ? solve(stdout) : reject(stderr)
+    })
+  })
+}
+
 /**Check command existance */
 export async function which(command) {
   try {

source/plugins/languages/analyzers.mjs

@@ -151,32 +151,31 @@ async function analyze({login, imports, data}, {results, path}) {
   console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking git log`)
   for (let page = 0; ; page++) {
     try {
-      const stdout = await imports.run(`git log ${data.shared["commits.authoring"].map(authoring => `--author="${authoring}"`).join(" ")} --regexp-ignore-case --format=short --patch --max-count=${per_page} --skip=${page*per_page}`, {cwd:path}, {log:false})
-      let file = null, lang = null
-      if (!stdout.trim().length) {
-        console.debug(`metrics/compute/${login}/plugins > languages > indepth > no more commits`)
-        break
-      }
       console.debug(`metrics/compute/${login}/plugins > languages > indepth > processing commits ${page*per_page} from ${(page+1)*per_page}`)
-      for (const line of stdout.split("\n").map(line => line.trim())) {
+      let file = null, lang = null, empty = true
+      await imports.spawn("git", ["log", ...data.shared["commits.authoring"].map(authoring => `--author="${authoring}"`), "--regexp-ignore-case", "--format=short", "--patch", `--max-count=${per_page}`, `--skip=${page*per_page}`], {cwd:path}, {
+        stdout(line) {
+          //Unflag empty output
+          if ((empty)&&(line.trim().length))
+            empty = false
           //Commits counter
           if (/^commit [0-9a-f]{40}$/.test(line)) {
             results.commits++
-          continue
+            return
           }
           //Ignore empty lines or unneeded lines
           if ((!/^[+]/.test(line))||(!line.length))
-          continue
+            return
           //File marker
           if (/^[+]{3}\sb[/](?<file>[\s\S]+)$/.test(line)) {
             file = line.match(/^[+]{3}\sb[/](?<file>[\s\S]+)$/)?.groups?.file ?? null
             lang = files[file] ?? null
             edited.add(file)
-          continue
+            return
           }
           //Ignore unkonwn languages
           if (!lang)
-          continue
+            return
           //Added line marker
           if (/^[+]\s*(?<line>[\s\S]+)$/.test(line)) {
             const size = Buffer.byteLength(line.match(/^[+]\s*(?<line>[\s\S]+)$/)?.groups?.line ?? "", "utf-8")
@@ -185,6 +184,11 @@ async function analyze({login, imports, data}, {results, path}) {
             results.total += size
           }
         }
+      })
+      if (empty) {
+        console.debug(`metrics/compute/${login}/plugins > languages > indepth > no more commits`)
+        break
+      }
     }
     catch {
       console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured on page ${page}, skipping...`)