Add plugin_languages_sections option and "recently-used" section (#327)

This commit is contained in:
Simon Lecoq
2021-05-25 23:22:57 +02:00
committed by GitHub
parent 2fd7452384
commit 7eb9afc589
5 changed files with 203 additions and 131 deletions

View File

@@ -0,0 +1,164 @@
/**Indepth analyzer */
export async function indepth({login, data, imports}, {skipped}) {
//Check prerequisites
if (!await imports.which("github-linguist"))
throw new Error("Feature requires github-linguist")
//Compute repositories stats from fetched repositories
const results = {total:0, stats:{}}
for (const repository of data.user.repositories.nodes) {
//Skip repository if asked
if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) {
console.debug(`metrics/compute/${login}/plugins > languages > skipped repository ${repository.owner.login}/${repository.name}`)
continue
}
//Repository handle
const repo = `${repository.owner.login}/${repository.name}`
console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking ${repo}`)
//Temporary directory
const path = imports.paths.join(imports.os.tmpdir(), `${data.user.databaseId}-${repo.replace(/[^\w]/g, "_")}`)
console.debug(`metrics/compute/${login}/plugins > languages > indepth > cloning ${repo} to temp dir ${path}`)
//Process
try {
//Git clone into temporary directory
await imports.fs.rmdir(path, {recursive:true})
await imports.fs.mkdir(path, {recursive:true})
const git = await imports.git(path)
await git.clone(`https://github.com/${repo}`, ".").status()
//Analyze repository
await analyze(arguments[0], {results, path})
}
catch {
console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured while processing ${repo}, skipping...`)
}
finally {
//Cleaning
console.debug(`metrics/compute/${login}/plugins > languages > indepth > cleaning temp dir ${path}`)
await imports.fs.rmdir(path, {recursive:true})
}
}
return results
}
/**Recent languages activity */
export async function recent({login, data, imports, rest, account}, {skipped}) {
//Check prerequisites
if (!await imports.which("github-linguist"))
throw new Error("Feature requires github-linguist")
//Get user recent activity
console.debug(`metrics/compute/${login}/plugins > languages > querying api`)
const commits = [], days = 14, pages = 3, results = {total:0, stats:{}}
try {
for (let page = 1; page <= pages; page++) {
console.debug(`metrics/compute/${login}/plugins > languages > loading page ${page}`)
commits.push(...(await rest.activity.listEventsForAuthenticatedUser({username:login, per_page:100, page})).data
.filter(({type}) => type === "PushEvent")
.filter(({actor}) => account === "organization" ? true : actor.login === login)
.filter(({repo:{name:repo}}) => (!skipped.includes(repo.toLocaleLowerCase())) && (!skipped.includes(repo.toLocaleLowerCase().split("/").pop())))
.filter(({created_at}) => new Date(created_at) > new Date(Date.now() - days * 24 * 60 * 60 * 1000))
)
}
}
catch {
console.debug(`metrics/compute/${login}/plugins > languages > no more page to load`)
}
console.debug(`metrics/compute/${login}/plugins > languages > ${commits.length} commits loaded`)
//Retrieve edited files and filter edited lines (those starting with +/-) from patches
console.debug(`metrics/compute/${login}/plugins > languages > loading patches`)
const patches = [
...await Promise.allSettled(
commits
.flatMap(({payload}) => payload.commits).map(commit => commit.url)
.map(async commit => (await rest.request(commit)).data.files),
),
]
.filter(({status}) => status === "fulfilled")
.map(({value}) => value)
.flatMap(files => files.map(file => ({name:imports.paths.basename(file.filename), patch:file.patch ?? ""})))
.map(({name, patch}) => ({name, patch:patch.split("\n").filter(line => /^[+]/.test(line)).map(line => line.substring(1)).join("\n")}))
//Temporary directory
const path = imports.paths.join(imports.os.tmpdir(), `${data.user.databaseId}`)
console.debug(`metrics/compute/${login}/plugins > languages > creating temp dir ${path} with ${patches.length} files`)
//Process
try {
//Save patches in temporary directory
await imports.fs.rmdir(path, {recursive:true})
await imports.fs.mkdir(path, {recursive:true})
await Promise.all(patches.map(({name, patch}, i) => imports.fs.writeFile(imports.paths.join(path, `${i}${imports.paths.extname(name)}`), patch)))
//Create temporary git repository
console.debug(`metrics/compute/${login}/plugins > languages > creating temp git repository`)
const git = await imports.git(path)
await git.init().add(".").addConfig("user.name", login).addConfig("user.email", "<>").commit("linguist").status()
//Analyze repository
await analyze(arguments[0], {results, path})
}
catch {
console.debug(`metrics/compute/${login}/plugins > languages > an error occured while processing recently used languages`)
}
finally {
//Cleaning
console.debug(`metrics/compute/${login}/plugins > languages > cleaning temp dir ${path}`)
await imports.fs.rmdir(path, {recursive:true})
}
console.log(results)
return results
}
/**Analyze a single repository */
async function analyze({login, imports}, {results, path}) {
//Spawn linguist process and map files to languages
console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`)
const files = Object.fromEntries(Object.entries(JSON.parse(await imports.run("github-linguist --json", {cwd:path}, {log:false}))).flatMap(([lang, files]) => files.map(file => [file, lang])))
console.log(files)
//Processing diff
const per_page = 10
console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking git log`)
for (let page = 0; ; page++) {
try {
const stdout = await imports.run(`git log --author="${login}" --format="" --patch --max-count=${per_page} --skip=${page*per_page}`, {cwd:path}, {log:false})
let file = null, lang = null
if (!stdout.trim().length) {
console.debug(`metrics/compute/${login}/plugins > languages > indepth > no more commits`)
break
}
console.debug(`metrics/compute/${login}/plugins > languages > indepth > processing commits ${page*per_page} from ${(page+1)*per_page}`)
for (const line of stdout.split("\n").map(line => line.trim())) {
//Ignore empty lines or unneeded lines
if ((!/^[+]/.test(line))||(!line.length))
continue
//File marker
if (/^[+]{3}\sb[/](?<file>[\s\S]+)$/.test(line)) {
file = line.match(/^[+]{3}\sb[/](?<file>[\s\S]+)$/)?.groups?.file ?? null
lang = files[file] ?? null
continue
}
//Ignore unkonwn languages
if (!lang)
continue
//Added line marker
if (/^[+]\s(?<line>[\s\S]+)$/.test(line)) {
const size = Buffer.byteLength(line.match(/^[+]\s(?<line>[\s\S]+)$/)?.groups?.line ?? "", "utf-8")
results.stats[lang] = (results.stats[lang] ?? 0) + size
results.total += size
}
}
}
catch {
console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured on page ${page}, skipping...`)
}
}
}

View File

@@ -1,105 +0,0 @@
/**Indepth analyzer */
export default async function({login, data, imports}, {skipped, ignored}) {
//Check prerequisites
if (!await imports.which("github-linguist"))
throw new Error("Feature requires github-linguist")
//Compute repositories stats
const results = {total:0, stats:{}}
for (const repository of data.user.repositories.nodes) {
const repo = `${repository.owner.login}/${repository.name}`
console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking ${repo}`)
//Skip repository if asked
if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) {
console.debug(`metrics/compute/${login}/plugins > languages > skipped repository ${repository.owner.login}/${repository.name}`)
continue
}
//Analyze
try {
await analyze(arguments[0], {repo, results})
}
catch {
console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured while processing ${repo}, skipping...`)
}
}
//Ignore languages if asked
Object.assign(results.stats, Object.fromEntries(Object.entries(results.stats).filter(([lang]) => !ignored.includes(lang.toLocaleLowerCase()))))
return results
}
/**Clone and analyze a single repository */
async function analyze({login, data, imports}, {repo, results}) {
//Git clone into a temporary directory
const path = imports.paths.join(imports.os.tmpdir(), `${data.user.databaseId}-${repo.replace(/[^\w]/g, "_")}`)
console.debug(`metrics/compute/${login}/plugins > languages > indepth > cloning ${repo} to temp dir ${path}`)
await imports.fs.rmdir(path, {recursive:true})
await imports.fs.mkdir(path, {recursive:true})
const git = await imports.git(path)
await git.clone(`https://github.com/${repo}`, ".").status()
//Spawn linguist process and map files to languages
console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`)
const files = {}
{
const stdout = await imports.run("github-linguist --breakdown", {cwd:path}, {log:false})
let lang = null
for (const line of stdout.split("\n").map(line => line.trim())) {
//Ignore empty lines
if (!line.length)
continue
//Language marker
if (/^(?<lang>[\s\S]+):\s*$/.test(line)) {
lang = line.match(/^(?<lang>[\s\S]+):\s*$/)?.groups?.lang ?? null
continue
}
//Store language
if (lang) {
files[line] = {lang}
continue
}
}
}
//Processing diff
const per_page = 10
console.debug(`metrics/compute/${login}/plugins > languages > indepth > checking git log`)
for (let page = 0; ; page++) {
try {
const stdout = await imports.run(`git log --author="${login}" --format="" --patch --max-count=${per_page} --skip=${page*per_page}`, {cwd:path}, {log:false})
let file = null, lang = null
if (!stdout.trim().length) {
console.debug(`metrics/compute/${login}/plugins > languages > indepth > no more commits`)
break
}
console.debug(`metrics/compute/${login}/plugins > languages > indepth > processing commits ${page*per_page} from ${(page+1)*per_page}`)
for (const line of stdout.split("\n").map(line => line.trim())) {
//Ignore empty lines or unneeded lines
if ((!/^[+]/.test(line))||(!line.length))
continue
//File marker
if (/^[+]{3}\sb[/](?<file>[\s\S]+)$/.test(line)) {
file = line.match(/^[+]{3}\sb[/](?<file>[\s\S]+)$/)?.groups?.file ?? null
lang = files[file]?.lang ?? null
continue
}
//Ignore unkonwn languages
if (!lang)
continue
//Added line marker
if (/^[+]\s(?<line>[\s\S]+)$/.test(line)) {
const size = Buffer.byteLength(line.match(/^[+]\s(?<line>[\s\S]+)$/)?.groups?.line ?? "", "utf-8")
results.stats[lang] = (results.stats[lang] ?? 0) + size
results.total += size
}
}
}
catch {
console.debug(`metrics/compute/${login}/plugins > languages > indepth > an error occured on page ${page}, skipping...`)
}
}
//Cleaning
console.debug(`metrics/compute/${login}/plugins > languages > indepth > cleaning temp dir ${path}`)
await imports.fs.rmdir(path, {recursive:true})
}

View File

@@ -1,8 +1,8 @@
//Imports
import indepth_analyzer from "./indepth.mjs"
import { indepth as indepth_analyzer, recent as recent_analyzer } from "./analyzers.mjs"
//Setup
export default async function({login, data, imports, q, account}, {enabled = false} = {}) {
export default async function({login, data, imports, q, rest, account}, {enabled = false} = {}) {
//Plugin execution
try {
//Check if plugin is enabled and requirements are met
@@ -10,7 +10,7 @@ export default async function({login, data, imports, q, account}, {enabled = fal
return null
//Load inputs
let {ignored, skipped, colors, details, threshold, limit, indepth} = imports.metadata.plugins.languages.inputs({data, account, q})
let {ignored, skipped, colors, details, threshold, limit, indepth, sections} = imports.metadata.plugins.languages.inputs({data, account, q})
threshold = (Number(threshold.replace(/%$/, "")) || 0) / 100
skipped.push(...data.shared["repositories.skipped"])
if (!limit)
@@ -25,7 +25,7 @@ export default async function({login, data, imports, q, account}, {enabled = fal
//Iterate through user's repositories and retrieve languages data
console.debug(`metrics/compute/${login}/plugins > languages > processing ${data.user.repositories.nodes.length} repositories`)
const languages = {details, colors:{}, total:0, stats:{}}
const languages = {sections, details, colors:{}, total:0, stats:{}, "stats.recent":{}}
for (const repository of data.user.repositories.nodes) {
//Skip repository if asked
if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) {
@@ -34,33 +34,35 @@ export default async function({login, data, imports, q, account}, {enabled = fal
}
//Process repository languages
for (const {size, node:{color, name}} of Object.values(repository.languages.edges)) {
//Ignore language if asked
if (ignored.includes(name.toLocaleLowerCase())) {
console.debug(`metrics/compute/${login}/plugins > languages > ignored language ${name}`)
continue
}
//Update language stats
languages.stats[name] = (languages.stats[name] ?? 0) + size
languages.colors[name] = colors[name.toLocaleLowerCase()] ?? color ?? "#ededed"
languages.total += size
}
}
//Recently used languages
if ((sections.includes("recently-used"))&&(["user", "organization"].includes(account))) {
console.debug(`metrics/compute/${login}/plugins > languages > using recent analyzer`)
languages["stats.recent"] = await recent_analyzer({login, data, imports, rest, account}, {skipped})
}
//Indepth mode
if (indepth) {
console.debug(`metrics/compute/${login}/plugins > languages > switching to indepth mode (this may take some time)`)
Object.assign(languages, await indepth_analyzer({login, data, imports}, {skipped, ignored}))
Object.assign(languages, await indepth_analyzer({login, data, imports}, {skipped}))
}
//Compute languages stats
console.debug(`metrics/compute/${login}/plugins > languages > computing stats`)
languages.favorites = Object.entries(languages.stats).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size:value, color:languages.colors[name], x:0})).filter(({value}) => value / languages.total > threshold)
const visible = {total:Object.values(languages.favorites).map(({size}) => size).reduce((a, b) => a + b, 0)}
for (let i = 0; i < languages.favorites.length; i++) {
languages.favorites[i].value /= visible.total
languages.favorites[i].x = (languages.favorites[i - 1]?.x ?? 0) + (languages.favorites[i - 1]?.value ?? 0)
if ((colors[i]) && (!colors[languages.favorites[i].name.toLocaleLowerCase()]))
languages.favorites[i].color = colors[i]
for (const {section, stats = {}, total = 0} of [{section:"favorites", stats:languages.stats, total:languages.total}, {section:"recent", ...languages["stats.recent"]}]) {
console.debug(`metrics/compute/${login}/plugins > languages > computing stats ${section}`)
languages[section] = Object.entries(stats).filter(([name]) => !ignored.includes(name.toLocaleLowerCase())).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size:value, color:languages.colors[name], x:0})).filter(({value}) => value / total > threshold)
const visible = {total:Object.values(languages[section]).map(({size}) => size).reduce((a, b) => a + b, 0)}
for (let i = 0; i < languages[section].length; i++) {
languages[section][i].value /= visible.total
languages[section][i].x = (languages[section][i - 1]?.x ?? 0) + (languages[section][i - 1]?.value ?? 0)
if ((colors[i]) && (!colors[languages[section][i].name.toLocaleLowerCase()]))
languages[section][i].color = colors[i]
}
}
//Results

View File

@@ -39,6 +39,17 @@ inputs:
min: 0
max: 8
# Sections to display
plugin_languages_sections:
description: Sections to display
type: array
format: comma-separated
default: most-used
example: most-used, recently-used
values:
- most-used # Most used languages
- recently-used # Recently used languages
# Overrides default languages colors
# Use `${n}:${color}` to change the color of the n-th most used language (e.g. "0:red" to make your most used language red)
# Use `${language}:${color}` to change the color of named language (e.g. "javascript:red" to make JavaScript language red, language case is ignored)

View File

@@ -1,8 +1,8 @@
<% if (plugins.languages) { %>
<% if (plugins.languages) { for (const section of plugins.languages.sections) { const languages = {"most-used":plugins.languages.favorites, "recently-used":plugins.languages.recent}[section] %>
<section class="column">
<h2 class="field">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path fill-rule="evenodd" d="M1.5 2.75a.25.25 0 01.25-.25h12.5a.25.25 0 01.25.25v8.5a.25.25 0 01-.25.25h-6.5a.75.75 0 00-.53.22L4.5 14.44v-2.19a.75.75 0 00-.75-.75h-2a.25.25 0 01-.25-.25v-8.5zM1.75 1A1.75 1.75 0 000 2.75v8.5C0 12.216.784 13 1.75 13H3v1.543a1.457 1.457 0 002.487 1.03L8.061 13h6.189A1.75 1.75 0 0016 11.25v-8.5A1.75 1.75 0 0014.25 1H1.75zm5.03 3.47a.75.75 0 010 1.06L5.31 7l1.47 1.47a.75.75 0 01-1.06 1.06l-2-2a.75.75 0 010-1.06l2-2a.75.75 0 011.06 0zm2.44 0a.75.75 0 000 1.06L10.69 7 9.22 8.47a.75.75 0 001.06 1.06l2-2a.75.75 0 000-1.06l-2-2a.75.75 0 00-1.06 0z"></path></svg>
Most used languages
<%= {"most-used":"Most used languages", "recently-used":"Recently used languages"}[section] %>
</h2>
<% if (plugins.languages.error) { %>
<section>
@@ -16,8 +16,8 @@
<mask id="languages-bar">
<rect x="0" y="0" width="<%= width %>" height="8" fill="white" rx="5"/>
</mask>
<rect mask="url(#languages-bar)" x="0" y="0" width="<%= plugins.languages.favorites.length ? 0 : width %>" height="8" fill="#d1d5da"/>
<% for (const {name, value, color, x} of plugins.languages.favorites) { %>
<rect mask="url(#languages-bar)" x="0" y="0" width="<%= languages.length ? 0 : width %>" height="8" fill="#d1d5da"/>
<% for (const {name, value, color, x} of languages) { %>
<rect mask="url(#languages-bar)" x="<%= x*width %>" y="0" width="<%= value*width %>" height="8" fill="<%= color %>"/>
<% } %>
</svg>
@@ -25,7 +25,7 @@
<div class="row fill-width">
<% for (const row of rows) { %>
<section>
<% for (const {name, value, color, size} of plugins.languages.favorites.filter((_, i) => i%rows.length === row)) { %>
<% for (const {name, value, color, size} of languages.filter((_, i) => i%rows.length === row)) { %>
<div class="field language details">
<div class="field">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path fill="<%= color %>" fill-rule="evenodd" d="M8 4a4 4 0 100 8 4 4 0 000-8z"></path></svg>
@@ -42,7 +42,7 @@
</div>
<% } else { %>
<div class="field center horizontal-wrap fill-width">
<% for (const {name, value, color} of plugins.languages.favorites) { %>
<% for (const {name, value, color} of languages) { %>
<div class="field center no-wrap language">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path fill="<%= color %>" fill-rule="evenodd" d="M8 4a4 4 0 100 8 4 4 0 000-8z"></path></svg>
<%= name %>
@@ -52,4 +52,4 @@
<% } %>
<% } %>
</section>
<% } %>
<% } } %>