From c13582ec528589062924ca19b9cd82e88aeffe31 Mon Sep 17 00:00:00 2001 From: lowlighter <22963968+lowlighter@users.noreply.github.com> Date: Thu, 17 Jun 2021 20:15:37 +0200 Subject: [PATCH] Use spawn instead of exec to handle large git log outputs --- source/app/metrics/utils.mjs | 31 ++++++++++-- source/plugins/languages/analyzers.mjs | 66 ++++++++++++++------------ 2 files changed, 63 insertions(+), 34 deletions(-) diff --git a/source/app/metrics/utils.mjs b/source/app/metrics/utils.mjs index 583b0b48..4dc4aefc 100644 --- a/source/app/metrics/utils.mjs +++ b/source/app/metrics/utils.mjs @@ -21,6 +21,7 @@ import twemojis from "twemoji-parser" import url from "url" import util from "util" import fetch from "node-fetch" +import readline from "readline" prism_lang() //Exports @@ -140,18 +141,18 @@ export async function chartist() { .replace(/class="ct-chart-line">/, `class="ct-chart-line">${css}`) } -/**Run command */ +/**Run command (use this to execute commands and process whole output at once, may not be suitable for large outputs) */ export async function run(command, options, {prefixed = true, log = true} = {}) { const prefix = {win32:"wsl"}[process.platform] ?? "" command = `${prefixed ? prefix : ""} ${command}`.trim() return new Promise((solve, reject) => { - console.debug(`metrics/command > ${command}`) + console.debug(`metrics/command/run > ${command}`) const child = processes.exec(command, options) let [stdout, stderr] = ["", ""] child.stdout.on("data", data => stdout += data) child.stderr.on("data", data => stderr += data) child.on("close", code => { - console.debug(`metrics/command > ${command} > exited with code ${code}`) + console.debug(`metrics/command/run > ${command} > exited with code ${code}`) if (log) { console.debug(stdout) console.debug(stderr) @@ -161,6 +162,30 @@ export async function run(command, options, {prefixed = true, log = true} = {}) }) } +/**Spawn command (use this to execute commands and process output on the fly) */ +export async function spawn(command, args = [], options = {}, {prefixed = true, timeout = 300*1000, stdout} = {}) { + const prefix = {win32:"wsl"}[process.platform] ?? "" + if ((prefixed)&&(prefix)) { + args.unshift(command) + command = prefix + } + if (!stdout) + throw new Error("`stdout` argument was not provided, use run() instead of spawn() if processing output is not needed") + return new Promise((solve, reject) => { + console.debug(`metrics/command/spawn > ${command} with ${args.join(" ")}`) + const child = processes.spawn(command, args, {...options, shell:true, timeout}) + const reader = readline.createInterface({input:child.stdout}) + reader.on("line", stdout) + const closed = new Promise(close => reader.on("close", close)) + child.on("close", async code => { + console.debug(`metrics/command/spawn > ${command} with ${args.join(" ")} > exited with code ${code}`) + await closed + console.debug(`metrics/command/spawn > ${command} with ${args.join(" ")} > reader closed`) + return code === 0 ? solve(stdout) : reject(stderr) + }) + }) +} + /**Check command existance */ export async function which(command) { try { diff --git a/source/plugins/languages/analyzers.mjs b/source/plugins/languages/analyzers.mjs index 9339e4f4..962402d2 100644 --- a/source/plugins/languages/analyzers.mjs +++ b/source/plugins/languages/analyzers.mjs @@ -151,40 +151,44 @@ async function analyze({login, imports, data}, {results, path}) { console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking git log`) for (let page = 0; ; page++) { try { - const stdout = await imports.run(`git log ${data.shared["commits.authoring"].map(authoring => `--author="${authoring}"`).join(" ")} --regexp-ignore-case --format=short --patch --max-count=${per_page} --skip=${page*per_page}`, {cwd:path}, {log:false}) - let file = null, lang = null - if (!stdout.trim().length) { + console.debug(`metrics/compute/${login}/plugins > languages > indepth > processing commits ${page*per_page} from ${(page+1)*per_page}`) + let file = null, lang = null, empty = true + await imports.spawn("git", ["log", ...data.shared["commits.authoring"].map(authoring => `--author="${authoring}"`), "--regexp-ignore-case", "--format=short", "--patch", `--max-count=${per_page}`, `--skip=${page*per_page}`], {cwd:path}, { + stdout(line) { + //Unflag empty output + if ((empty)&&(line.trim().length)) + empty = false + //Commits counter + if (/^commit [0-9a-f]{40}$/.test(line)) { + results.commits++ + return + } + //Ignore empty lines or unneeded lines + if ((!/^[+]/.test(line))||(!line.length)) + return + //File marker + if (/^[+]{3}\sb[/](?[\s\S]+)$/.test(line)) { + file = line.match(/^[+]{3}\sb[/](?[\s\S]+)$/)?.groups?.file ?? null + lang = files[file] ?? null + edited.add(file) + return + } + //Ignore unkonwn languages + if (!lang) + return + //Added line marker + if (/^[+]\s*(?[\s\S]+)$/.test(line)) { + const size = Buffer.byteLength(line.match(/^[+]\s*(?[\s\S]+)$/)?.groups?.line ?? "", "utf-8") + results.stats[lang] = (results.stats[lang] ?? 0) + size + results.lines[lang] = (results.lines[lang] ?? 0) + 1 + results.total += size + } + } + }) + if (empty) { console.debug(`metrics/compute/${login}/plugins > languages > indepth > no more commits`) break } - console.debug(`metrics/compute/${login}/plugins > languages > indepth > processing commits ${page*per_page} from ${(page+1)*per_page}`) - for (const line of stdout.split("\n").map(line => line.trim())) { - //Commits counter - if (/^commit [0-9a-f]{40}$/.test(line)) { - results.commits++ - continue - } - //Ignore empty lines or unneeded lines - if ((!/^[+]/.test(line))||(!line.length)) - continue - //File marker - if (/^[+]{3}\sb[/](?[\s\S]+)$/.test(line)) { - file = line.match(/^[+]{3}\sb[/](?[\s\S]+)$/)?.groups?.file ?? null - lang = files[file] ?? null - edited.add(file) - continue - } - //Ignore unkonwn languages - if (!lang) - continue - //Added line marker - if (/^[+]\s*(?[\s\S]+)$/.test(line)) { - const size = Buffer.byteLength(line.match(/^[+]\s*(?[\s\S]+)$/)?.groups?.line ?? "", "utf-8") - results.stats[lang] = (results.stats[lang] ?? 0) + size - results.lines[lang] = (results.lines[lang] ?? 0) + 1 - results.total += size - } - } } catch { console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured on page ${page}, skipping...`)