Replace github-linguist gem with Node port Nixinova/Linguist (#436)

Dockerfile

@@ -19,7 +19,6 @@ RUN chmod +x /metrics/source/app/action/index.mjs \
   # Based on https://github.com/github/linguist and https://github.com/github/licensed
   && apt-get install -y ruby-full \
   && apt-get install -y git g++ cmake pkg-config libicu-dev zlib1g-dev libcurl4-openssl-dev libssl-dev ruby-dev \
-  && gem install github-linguist -v 7.15.0 \
   && gem install licensed \
   # Install python for node-gyp
   && apt-get install -y python3 \
@@ -35,4 +34,4 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
 ENV PUPPETEER_BROWSER_PATH "google-chrome-stable"
 
 # Execute GitHub action
-ENTRYPOINT node /metrics/source/app/action/index.mjs
+ENTRYPOINT node /metrics/source/app/action/index.mjs

package-lock.json

@@ -25,6 +25,7 @@
         "faker": "^5.5.3",
         "jimp": "^0.16.1",
         "js-yaml": "^4.1.0",
+        "linguist-js": "^1.4.3",
         "marked": "^2.1.3",
         "memory-cache": "^0.2.0",
         "node-chartist": "^1.0.5",
@@ -4672,6 +4673,11 @@
         "node": ">= 6"
       }
     },
+    "node_modules/glob-to-regexp": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
+      "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw=="
+    },
     "node_modules/global": {
       "version": "4.4.0",
       "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz",
@@ -6335,6 +6341,21 @@
         "node": ">=10"
       }
     },
+    "node_modules/linguist-js": {
+      "version": "1.4.3",
+      "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.3.tgz",
+      "integrity": "sha512-lAfJS6ZIlmCfZmefIUUwYO2Bdr6eIPcAVurEbOXR6KaXDmEH9ATiI/SOrnN9djTuatJQG7AnqJUSmud+7KHHrA==",
+      "dependencies": {
+        "glob": "^7.1.7",
+        "glob-to-regexp": "^0.4.1",
+        "js-yaml": "^4.1.0",
+        "node-fetch": "^2.6.1",
+        "yargs-parser": "^20.2.9"
+      },
+      "bin": {
+        "linguist": "bin/index.js"
+      }
+    },
     "node_modules/load-bmfont": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz",
@@ -9899,10 +9920,9 @@
       }
     },
     "node_modules/yargs-parser": {
-      "version": "20.2.7",
+      "version": "20.2.9",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz",
-      "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==",
+      "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==",
-      "dev": true,
       "engines": {
         "node": ">=10"
       }
@@ -13581,6 +13601,11 @@
         "is-glob": "^4.0.1"
       }
     },
+    "glob-to-regexp": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
+      "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw=="
+    },
     "global": {
       "version": "4.4.0",
       "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz",
@@ -14852,6 +14877,18 @@
         "nan": "~2.14.0"
       }
     },
+    "linguist-js": {
+      "version": "1.4.3",
+      "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.3.tgz",
+      "integrity": "sha512-lAfJS6ZIlmCfZmefIUUwYO2Bdr6eIPcAVurEbOXR6KaXDmEH9ATiI/SOrnN9djTuatJQG7AnqJUSmud+7KHHrA==",
+      "requires": {
+        "glob": "^7.1.7",
+        "glob-to-regexp": "^0.4.1",
+        "js-yaml": "^4.1.0",
+        "node-fetch": "^2.6.1",
+        "yargs-parser": "^20.2.9"
+      }
+    },
     "load-bmfont": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz",
@@ -17687,10 +17724,9 @@
       }
     },
     "yargs-parser": {
-      "version": "20.2.7",
+      "version": "20.2.9",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz",
-      "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==",
+      "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w=="
-      "dev": true
     },
     "yauzl": {
       "version": "2.10.0",

package.json

@@ -41,6 +41,7 @@
     "faker": "^5.5.3",
     "jimp": "^0.16.1",
     "js-yaml": "^4.1.0",
+    "linguist-js": "^1.4.3",
     "marked": "^2.1.3",
     "memory-cache": "^0.2.0",
     "node-chartist": "^1.0.5",

source/plugins/habits/index.mjs

@@ -100,7 +100,7 @@ export default async function({login, data, rest, imports, q, account}, {enabled
     if (charts) {
       //Check if linguist exists
       console.debug(`metrics/compute/${login}/plugins > habits > searching recently used languages using linguist`)
-      if ((patches.length) && (await imports.which("github-linguist"))) {
+      if (patches.length) {
         //Call language analyzer (note: using content from other plugin is usually disallowed, this is mostly for legacy purposes)
         habits.linguist.available = true
         const {total, stats} = await recent_analyzer({login, data, imports, rest, account}, {days, load:from || 1000, tempdir:"habits"})

source/plugins/languages/analyzers.mjs

@@ -1,8 +1,7 @@
+import linguist from "linguist-js"
+
 /**Indepth analyzer */
 export async function indepth({login, data, imports, repositories}, {skipped}) {
-  //Check prerequisites
-  if (!await imports.which("github-linguist"))
-    throw new Error("Feature requires github-linguist")
 
   //Compute repositories stats from fetched repositories
   const results = {total:0, lines:{}, stats:{}, commits:0, files:0, missed:0}
@@ -46,9 +45,6 @@ export async function indepth({login, data, imports, repositories}, {skipped}) {
 
 /**Recent languages activity */
 export async function recent({login, data, imports, rest, account}, {skipped = [], days = 0, load = 0, tempdir = "recent"}) {
-  //Check prerequisites
-  if (!await imports.which("github-linguist"))
-    throw new Error("Feature requires github-linguist")
 
   //Get user recent activity
   console.debug(`metrics/compute/${login}/plugins > languages > querying api`)
@@ -141,9 +137,9 @@ export async function recent({login, data, imports, rest, account}, {skipped = [
 
 /**Analyze a single repository */
 async function analyze({login, imports, data}, {results, path}) {
-  //Spawn linguist process and map files to languages
+  //Gather language data
   console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`)
-  const files = Object.fromEntries(Object.entries(JSON.parse(await imports.run("github-linguist --json", {cwd:path}, {log:false}))).flatMap(([lang, files]) => files.map(file => [file, lang])))
+  const {results:files, languages:languageResults} = await linguist(path)
 
   //Processing diff
   const per_page = 1
@@ -168,8 +164,10 @@ async function analyze({login, imports, data}, {results, path}) {
             return
           //File marker
           if (/^[+]{3}\sb[/](?<file>[\s\S]+)$/.test(line)) {
-            file = line.match(/^[+]{3}\sb[/](?<file>[\s\S]+)$/)?.groups?.file ?? null
+            file = line.match(/^[+]{3}\sb[/](?<file>[\s\S]+)$/)?.groups?.file.replace(/^/, `${path}/`) ?? null
             lang = files[file] ?? null
+            if (lang in languageResults.data || lang in languageResults.prose)
+              lang = null
             edited.add(file)
             return
           }