Replace github-linguist gem with Node port Nixinova/Linguist (#436)

2021-08-01 22:54:03 +12:00
parent fbd0944a55
commit 9e77a1b2c0
5 changed files with 54 additions and 20 deletions
--- a/1
+++ b/1
@@ -19,7 +19,6 @@ RUN chmod +x /metrics/source/app/action/index.mjs \
  # Based on https://github.com/github/linguist and https://github.com/github/licensed
  && apt-get install -y ruby-full \
  && apt-get install -y git g++ cmake pkg-config libicu-dev zlib1g-dev libcurl4-openssl-dev libssl-dev ruby-dev \
-  && gem install github-linguist -v 7.15.0 \
  && gem install licensed \
  # Install python for node-gyp
  && apt-get install -y python3 \
--- a/package-lock.json
+++ b/package-lock.json
@@ -25,6 +25,7 @@
        "faker": "^5.5.3",
        "jimp": "^0.16.1",
        "js-yaml": "^4.1.0",
+        "linguist-js": "^1.4.3",
        "marked": "^2.1.3",
        "memory-cache": "^0.2.0",
        "node-chartist": "^1.0.5",
@@ -4672,6 +4673,11 @@
        "node": ">= 6"
      }
    },
+    "node_modules/glob-to-regexp": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
+      "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw=="
+    },
    "node_modules/global": {
      "version": "4.4.0",
      "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz",
@@ -6335,6 +6341,21 @@
        "node": ">=10"
      }
    },
+    "node_modules/linguist-js": {
+      "version": "1.4.3",
+      "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.3.tgz",
+      "integrity": "sha512-lAfJS6ZIlmCfZmefIUUwYO2Bdr6eIPcAVurEbOXR6KaXDmEH9ATiI/SOrnN9djTuatJQG7AnqJUSmud+7KHHrA==",
+      "dependencies": {
+        "glob": "^7.1.7",
+        "glob-to-regexp": "^0.4.1",
+        "js-yaml": "^4.1.0",
+        "node-fetch": "^2.6.1",
+        "yargs-parser": "^20.2.9"
+      },
+      "bin": {
+        "linguist": "bin/index.js"
+      }
+    },
    "node_modules/load-bmfont": {
      "version": "1.4.1",
      "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz",
@@ -9899,10 +9920,9 @@
      }
    },
    "node_modules/yargs-parser": {
-      "version": "20.2.7",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz",
-      "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==",
-      "dev": true,
+      "version": "20.2.9",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz",
+      "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==",
      "engines": {
        "node": ">=10"
      }
@@ -13581,6 +13601,11 @@
        "is-glob": "^4.0.1"
      }
    },
+    "glob-to-regexp": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
+      "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw=="
+    },
    "global": {
      "version": "4.4.0",
      "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz",
@@ -14852,6 +14877,18 @@
        "nan": "~2.14.0"
      }
    },
+    "linguist-js": {
+      "version": "1.4.3",
+      "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.3.tgz",
+      "integrity": "sha512-lAfJS6ZIlmCfZmefIUUwYO2Bdr6eIPcAVurEbOXR6KaXDmEH9ATiI/SOrnN9djTuatJQG7AnqJUSmud+7KHHrA==",
+      "requires": {
+        "glob": "^7.1.7",
+        "glob-to-regexp": "^0.4.1",
+        "js-yaml": "^4.1.0",
+        "node-fetch": "^2.6.1",
+        "yargs-parser": "^20.2.9"
+      }
+    },
    "load-bmfont": {
      "version": "1.4.1",
      "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz",
@@ -17687,10 +17724,9 @@
      }
    },
    "yargs-parser": {
-      "version": "20.2.7",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz",
-      "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==",
-      "dev": true
+      "version": "20.2.9",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz",
+      "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w=="
    },
    "yauzl": {
      "version": "2.10.0",
--- a/package.json
+++ b/package.json
@@ -41,6 +41,7 @@
    "faker": "^5.5.3",
    "jimp": "^0.16.1",
    "js-yaml": "^4.1.0",
+    "linguist-js": "^1.4.3",
    "marked": "^2.1.3",
    "memory-cache": "^0.2.0",
    "node-chartist": "^1.0.5",
--- a/source/plugins/habits/index.mjs
+++ b/source/plugins/habits/index.mjs
@@ -100,7 +100,7 @@ export default async function({login, data, rest, imports, q, account}, {enabled
    if (charts) {
      //Check if linguist exists
      console.debug(`metrics/compute/${login}/plugins > habits > searching recently used languages using linguist`)
-      if ((patches.length) && (await imports.which("github-linguist"))) {
+      if (patches.length) {
        //Call language analyzer (note: using content from other plugin is usually disallowed, this is mostly for legacy purposes)
        habits.linguist.available = true
        const {total, stats} = await recent_analyzer({login, data, imports, rest, account}, {days, load:from || 1000, tempdir:"habits"})
--- a/source/plugins/languages/analyzers.mjs
+++ b/source/plugins/languages/analyzers.mjs
@@ -1,8 +1,7 @@
+import linguist from "linguist-js"
+
 /**Indepth analyzer */
 export async function indepth({login, data, imports, repositories}, {skipped}) {
-  //Check prerequisites
-  if (!await imports.which("github-linguist"))
-    throw new Error("Feature requires github-linguist")

  //Compute repositories stats from fetched repositories
  const results = {total:0, lines:{}, stats:{}, commits:0, files:0, missed:0}
@@ -46,9 +45,6 @@ export async function indepth({login, data, imports, repositories}, {skipped}) {

 /**Recent languages activity */
 export async function recent({login, data, imports, rest, account}, {skipped = [], days = 0, load = 0, tempdir = "recent"}) {
-  //Check prerequisites
-  if (!await imports.which("github-linguist"))
-    throw new Error("Feature requires github-linguist")

  //Get user recent activity
  console.debug(`metrics/compute/${login}/plugins > languages > querying api`)
@@ -141,9 +137,9 @@ export async function recent({login, data, imports, rest, account}, {skipped = [

 /**Analyze a single repository */
 async function analyze({login, imports, data}, {results, path}) {
-  //Spawn linguist process and map files to languages
+  //Gather language data
  console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`)
-  const files = Object.fromEntries(Object.entries(JSON.parse(await imports.run("github-linguist --json", {cwd:path}, {log:false}))).flatMap(([lang, files]) => files.map(file => [file, lang])))
+  const {results:files, languages:languageResults} = await linguist(path)

  //Processing diff
  const per_page = 1
@@ -168,8 +164,10 @@ async function analyze({login, imports, data}, {results, path}) {
            return
          //File marker
          if (/^[+]{3}\sb[/](?<file>[\s\S]+)$/.test(line)) {
-            file = line.match(/^[+]{3}\sb[/](?<file>[\s\S]+)$/)?.groups?.file ?? null
+            file = line.match(/^[+]{3}\sb[/](?<file>[\s\S]+)$/)?.groups?.file.replace(/^/, `${path}/`) ?? null
            lang = files[file] ?? null
+            if (lang in languageResults.data || lang in languageResults.prose)
+              lang = null
            edited.add(file)
            return
          }