Reproduce partial repositories and fetch gitattributes for better recent languages (#356)

source/plugins/habits/index.mjs

@@ -1,3 +1,6 @@
+//Legacy import
+import { recent as recent_analyzer } from "./../languages/analyzers.mjs"
+
 //Setup
 export default async function({login, data, rest, imports, q, account}, {enabled = false, ...defaults} = {}) {
   //Plugin execution
@@ -97,27 +100,11 @@ export default async function({login, data, rest, imports, q, account}, {enabled
       //Check if linguist exists
       console.debug(`metrics/compute/${login}/plugins > habits > searching recently used languages using linguist`)
       if ((patches.length) && (await imports.which("github-linguist"))) {
-        //Setup for linguist
+        //Call language analyzer (note: using content from other plugin is usually disallowed, this is mostly for legacy purposes)
         habits.linguist.available = true
-        const path = imports.paths.join(imports.os.tmpdir(), `${commits[0]?.actor?.id ?? 0}`)
+        const {total, stats} = await recent_analyzer({login, data, imports, rest, account}, {days, load:from || 1000})
-        //Create temporary directory and save patches
+        habits.linguist.languages = Object.fromEntries(Object.entries(stats).map(([language, value]) => [language, value/total]))
-        console.debug(`metrics/compute/${login}/plugins > habits > creating temp dir ${path} with ${patches.length} files`)
-        await imports.fs.mkdir(path, {recursive:true})
-        await Promise.all(patches.map(({name, patch}, i) => imports.fs.writeFile(imports.paths.join(path, `${i}${imports.paths.extname(name)}`), patch)))
-        //Create temporary git repository
-        console.debug(`metrics/compute/${login}/plugins > habits > creating temp git repository`)
-        const git = await imports.git(path)
-        await git.init().add(".").addConfig("user.name", "linguist").addConfig("user.email", "<>").commit("linguist").status()
-        //Spawn linguist process
-        console.debug(`metrics/compute/${login}/plugins > habits > running linguist`)
-        ;(await imports.run("github-linguist --breakdown", {cwd:path}))
-          //Parse linguist result
-          .split("\n").map(line => line.match(/(?<value>[\d.]+)%\s+(?<language>[\s\S]+)$/)?.groups).filter(line => line)
-          .map(({value, language}) => habits.linguist.languages[language] = (habits.linguist.languages[language] ?? 0) + value / 100)
         habits.linguist.ordered = Object.entries(habits.linguist.languages).sort(([_an, a], [_bn, b]) => b - a)
-        //Cleaning
-        console.debug(`metrics/compute/${login}/plugins > habits > cleaning temp dir ${path}`)
-        await imports.fs.rmdir(path, {recursive:true})
       }
       else
         console.debug(`metrics/compute/${login}/plugins > habits > linguist not available`)

source/plugins/languages/analyzers.mjs

@@ -45,7 +45,7 @@ export async function indepth({login, data, imports, repositories}, {skipped}) {
 }
 
 /**Recent languages activity */
-export async function recent({login, data, imports, rest, account}, {skipped, days = 0, load = 0}) {
+export async function recent({login, data, imports, rest, account}, {skipped = [], days = 0, load = 0}) {
   //Check prerequisites
   if (!await imports.which("github-linguist"))
     throw new Error("Feature requires github-linguist")
@@ -85,8 +85,8 @@ export async function recent({login, data, imports, rest, account}, {skipped, da
   ]
   .filter(({status}) => status === "fulfilled")
   .map(({value}) => value)
-  .flatMap(files => files.map(file => ({name:imports.paths.basename(file.filename), patch:file.patch ?? ""})))
+  .flatMap(files => files.map(file => ({name:imports.paths.basename(file.filename), directory:imports.paths.dirname(file.filename), patch:file.patch ?? "", repo:file.raw_url.match(/(?<=^https:..github.com\/)(?<repo>.*)(?=\/raw)/)?.groups.repo ?? "_"})))
-  .map(({name, patch}) => ({name, patch:patch.split("\n").filter(line => /^[+]/.test(line)).map(line => line.substring(1)).join("\n")}))
+  .map(({name, directory, patch, repo}) => ({name, directory:`${repo.replace(/[/]/g, "@")}/${directory}`, patch:patch.split("\n").filter(line => /^[+]/.test(line)).map(line => line.substring(1)).join("\n")}))
 
   //Temporary directory
   const path = imports.paths.join(imports.os.tmpdir(), `${data.user.databaseId}`)
@@ -95,18 +95,37 @@ export async function recent({login, data, imports, rest, account}, {skipped, da
 
   //Process
   try {
-    //Save patches in temporary directory
+    //Save patches in temporary directory matching respective repository and filename
     await imports.fs.rmdir(path, {recursive:true})
     await imports.fs.mkdir(path, {recursive:true})
-    await Promise.all(patches.map(({name, patch}, i) => imports.fs.writeFile(imports.paths.join(path, `${i}${imports.paths.extname(name)}`), patch)))
+    await Promise.all(patches.map(async ({name, directory, patch}) => {
+      await imports.fs.mkdir(imports.paths.join(path, directory), {recursive:true})
+      imports.fs.writeFile(imports.paths.join(path, directory, name), patch)
+    }))
 
-    //Create temporary git repository
+    //Process temporary repositories
-    console.debug(`metrics/compute/${login}/plugins > languages > creating temp git repository`)
+    for (const directory of await imports.fs.readdir(path)) {
-    const git = await imports.git(path)
+      //Pull gitattributes if possible
-    await git.init().add(".").addConfig("user.name", data.shared["commits.authoring"]?.[0] ?? login).addConfig("user.email", "<>").commit("linguist").status()
+      for (const branch of ["main", "master"]) {
+        const repo = directory.replace("@", "/")
+        try {
+          await imports.fs.writeFile(imports.paths.join(path, directory, ".gitattributes"), await imports.fetch(`https://raw.githubusercontent.com/${repo}/${branch}/.gitattributes`).then(response => response.text()).catch(() => ""))
+          console.debug(`metrics/compute/${login}/plugins > languages > successfully fetched .gitattributes for ${repo}`)
+          break
+        }
+        catch {
+          console.debug(`metrics/compute/${login}/plugins > languages > cannot load .gitattributes on branch ${branch} for ${repo}`)
+        }
+      }
 
-    //Analyze repository
+      //Create temporary git repository
-    await analyze(arguments[0], {results, path})
+      console.debug(`metrics/compute/${login}/plugins > languages > creating temp git repository for ${directory}`)
+      const git = await imports.git(path)
+      await git.init().add(".").addConfig("user.name", data.shared["commits.authoring"]?.[0] ?? login).addConfig("user.email", "<>").commit("linguist").status()
+
+      //Analyze repository
+      await analyze(arguments[0], {results, path})
+    }
   }
   catch {
     console.debug(`metrics/compute/${login}/plugins > languages > an error occured while processing recently used languages`)