ref(plugins): centralize skip and ignore filters (#1238)

source/app/metrics/utils.mjs

@@ -323,51 +323,90 @@ export async function markdown(text, {mode = "inline", codelines = Infinity} = {
   return rendered
 }
 
-/**Check GitHub filter against object */
+/**Filters */
-export function ghfilter(text, object) {
+export const filters = {
-  console.debug(`metrics/svg/ghquery > checking ${text} against ${JSON.stringify(object)}`)
+  /**GitHub query filter */
-  const result = text.split(/(?<!NOT) /).map(x => x.trim()).filter(x => x).map(criteria => {
+  github(text, object) {
-    const [key, filters] = criteria.split(":")
+    console.debug(`metrics/svg/ghquery > checking ${text} against ${JSON.stringify(object)}`)
-    const value = object[/^NOT /.test(key) ? key.substring(3).trim() : /^-/.test(key) ? key.substring(1).trim() : key.trim()]
+    const result = text.split(/(?<!NOT) /).map(x => x.trim()).filter(x => x).map(criteria => {
-    console.debug(`metrics/svg/ghquery > checking ${criteria} against ${value}`)
+      const [key, filters] = criteria.split(":")
-    if (value === undefined) {
+      const value = object[/^NOT /.test(key) ? key.substring(3).trim() : /^-/.test(key) ? key.substring(1).trim() : key.trim()]
-      console.debug(`metrics/svg/ghquery > value for ${criteria} is undefined, considering it truthy`)
+      console.debug(`metrics/svg/ghquery > checking ${criteria} against ${value}`)
-      return true
+      if (value === undefined) {
-    }
+        console.debug(`metrics/svg/ghquery > value for ${criteria} is undefined, considering it truthy`)
-    return filters?.split(",").map(x => x.trim()).filter(x => x).map(filter => {
+        return true
-      if (!Number.isFinite(Number(value))) {
-        if (/^NOT /.test(filter))
-          return value !== filter.substring(3).trim()
-        if (/^-/.test(key))
-          return value !== filter
-        return value === filter.trim()
       }
-      switch (true) {
+      return filters?.split(",").map(x => x.trim()).filter(x => x).map(filter => {
-        case /^true$/.test(filter):
+        if (!Number.isFinite(Number(value))) {
-          return value === true
+          if (/^NOT /.test(filter))
-        case /^false$/.test(filter):
+            return value !== filter.substring(3).trim()
-          return value === false
+          if (/^-/.test(key))
-        case /^>\d+$/.test(filter):
+            return value !== filter
-          return value > Number(filter.substring(1))
+          return value === filter.trim()
-        case /^>=\d+$/.test(filter):
-          return value >= Number(filter.substring(2))
-        case /^<\d+$/.test(filter):
-          return value < Number(filter.substring(1))
-        case /^<=\d+$/.test(filter):
-          return value <= Number(filter.substring(2))
-        case /^\d+$/.test(filter):
-          return value === Number(filter)
-        case /^\d+..\d+$/.test(filter): {
-          const [a, b] = filter.split("..").map(Number)
-          return (value >= a) && (value <= b)
         }
-        default:
+        switch (true) {
-          return false
+          case /^true$/.test(filter):
-      }
+            return value === true
-    }).reduce((a, b) => a || b, false) ?? false
+          case /^false$/.test(filter):
-  }).reduce((a, b) => a && b, true)
+            return value === false
-  console.debug(`metrics/svg/ghquery > ${result ? "matching" : "not matching"}`)
+          case /^>\d+$/.test(filter):
-  return result
+            return value > Number(filter.substring(1))
+          case /^>=\d+$/.test(filter):
+            return value >= Number(filter.substring(2))
+          case /^<\d+$/.test(filter):
+            return value < Number(filter.substring(1))
+          case /^<=\d+$/.test(filter):
+            return value <= Number(filter.substring(2))
+          case /^\d+$/.test(filter):
+            return value === Number(filter)
+          case /^\d+..\d+$/.test(filter): {
+            const [a, b] = filter.split("..").map(Number)
+            return (value >= a) && (value <= b)
+          }
+          default:
+            return false
+        }
+      }).reduce((a, b) => a || b, false) ?? false
+    }).reduce((a, b) => a && b, true)
+    console.debug(`metrics/svg/ghquery > ${result ? "matching" : "not matching"}`)
+    return result
+  },
+  /**Repository filter*/
+  repo(repository, patterns) {
+    //Disable filtering when no pattern is provided
+    if (!patterns.length)
+      return true
+
+    //Normalize repository handle
+    let repo, user
+    if (repository.nameWithOwner)
+      repository = repository.nameWithOwner
+    if ((repository.name)&&(repository.owner?.login)) {
+      user = repository.owner.login
+      repo = repository.name
+    }
+    user = (user ?? repository.split("/")[0]).toLocaleLowerCase()
+    repo = (repo ?? repository.split("/")[1]).toLocaleLowerCase()
+
+    //Basic pattern matching
+    const include = (!patterns.includes(repo)) && (!patterns.includes(`${user}/${repo}`))
+    console.debug(`metrics/filters/repo > filter ${repo} (${include ? "included" : "excluded"})`)
+    return include
+  },
+  /**Text filter*/
+  text(text, patterns) {
+    //Disable filtering when no pattern is provided
+    if (!patterns.length)
+      return true
+
+    //Normalize text
+    text = `${text}`.toLocaleLowerCase()
+
+    //Basic pattern matching
+    const include = !patterns.includes(text)
+    console.debug(`metrics/filters/text > filter ${text} (${include ? "included" : "excluded"})`)
+    return include
+  }
 }
 
 /**Image to base64 */

source/plugins/achievements/index.mjs

@@ -23,7 +23,7 @@ export default async function({login, q, imports, data, computed, graphql, queri
     const achievements = list
       .filter(a => (order[a.rank] >= order[threshold]) || ((a.rank === "$") && (secrets)))
       .filter(a => (!only.length) || ((only.length) && (only.includes(a.title.toLocaleLowerCase()))))
-      .filter(a => !ignored.includes(a.title.toLocaleLowerCase()))
+      .filter(a => imports.filters.text(a.title, ignored))
       .sort((a, b) => (order[b.rank] + b.progress * 0.99) - (order[a.rank] + a.progress * 0.99))
       .map(({title, unlock, ...achievement}) => ({
         prefix: ({S: "Master", A: "Super", B: "Great"}[achievement.rank] ?? ""),

source/plugins/activity/index.mjs

@@ -46,7 +46,7 @@ export default async function({login, data, rest, q, account, imports}, {enabled
         .map(async ({type, payload, actor: {login: actor}, repo: {name: repo}, created_at}) => {
           //See https://docs.github.com/en/free-pro-team@latest/developers/webhooks-and-events/github-event-types
           const timestamp = new Date(created_at)
-          if ((skipped.includes(repo.split("/").pop())) || (skipped.includes(repo)))
+          if (!imports.filters.repo(repo, skipped))
             return null
           switch (type) {
             //Commented on a commit
@@ -54,7 +54,7 @@ export default async function({login, data, rest, q, account, imports}, {enabled
               if (!["created"].includes(payload.action))
                 return null
               const {comment: {user: {login: user}, commit_id: sha, body: content}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                 return null
               return {type: "comment", on: "commit", actor, timestamp, repo, content: await imports.markdown(content, {mode: markdown, codelines}), user, mobile: null, number: sha.substring(0, 7), title: ""}
             }
@@ -83,7 +83,7 @@ export default async function({login, data, rest, q, account, imports}, {enabled
               if (!["created"].includes(payload.action))
                 return null
               const {issue: {user: {login: user}, title, number}, comment: {body: content, performed_via_github_app: mobile}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                 return null
               return {type: "comment", on: "issue", actor, timestamp, repo, content: await imports.markdown(content, {mode: markdown, codelines}), user, mobile, number, title}
             }
@@ -92,7 +92,7 @@ export default async function({login, data, rest, q, account, imports}, {enabled
               if (!["opened", "closed", "reopened"].includes(payload.action))
                 return null
               const {action, issue: {user: {login: user}, title, number, body: content}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                 return null
               return {type: "issue", actor, timestamp, repo, action, user, number, title, content: await imports.markdown(content, {mode: markdown, codelines})}
             }
@@ -101,7 +101,7 @@ export default async function({login, data, rest, q, account, imports}, {enabled
               if (!["added"].includes(payload.action))
                 return null
               const {member: {login: user}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                 return null
               return {type: "member", actor, timestamp, repo, user}
             }
@@ -114,14 +114,14 @@ export default async function({login, data, rest, q, account, imports}, {enabled
               if (!["opened", "closed"].includes(payload.action))
                 return null
               const {action, pull_request: {user: {login: user}, title, number, body: content, additions: added, deletions: deleted, changed_files: changed, merged}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                 return null
               return {type: "pr", actor, timestamp, repo, action: (action === "closed") && (merged) ? "merged" : action, user, title, number, content: await imports.markdown(content, {mode: markdown, codelines}), lines: {added, deleted}, files: {changed}}
             }
             //Reviewed a pull request
             case "PullRequestReviewEvent": {
               const {review: {state: review}, pull_request: {user: {login: user}, number, title}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                 return null
               return {type: "review", actor, timestamp, repo, review, user, number, title}
             }
@@ -130,14 +130,14 @@ export default async function({login, data, rest, q, account, imports}, {enabled
               if (!["created"].includes(payload.action))
                 return null
               const {pull_request: {user: {login: user}, title, number}, comment: {body: content, performed_via_github_app: mobile}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                 return null
               return {type: "comment", on: "pr", actor, timestamp, repo, content: await imports.markdown(content, {mode: markdown, codelines}), user, mobile, number, title}
             }
             //Pushed commits
             case "PushEvent": {
               let {size, commits, ref} = payload
-              commits = commits.filter(({author: {email}}) => !ignored.includes(email))
+              commits = commits.filter(({author: {email}}) => imports.filters.text(email, ignored))
               if (!commits.length)
                 return null
               if (commits.slice(-1).pop()?.message.startsWith("Merge branch "))

source/plugins/code/index.mjs

@@ -35,7 +35,7 @@ export default async function({login, q, imports, data, rest, account}, {enabled
                 : await rest.activity.listEventsForAuthenticatedUser({username: login, per_page: 100, page})).data
                 .filter(({type}) => type === "PushEvent")
                 .filter(({actor}) => account === "organization" ? true : actor.login?.toLocaleLowerCase() === login.toLocaleLowerCase())
-                .filter(({repo: {name: repo}}) => !((skipped.includes(repo.split("/").pop())) || (skipped.includes(repo))))
+                .filter(({repo: {name: repo}}) => imports.filters.repo(repo, skipped))
                 .filter(event => visibility === "public" ? event.public : true)
                 .filter(({created_at}) => Number.isFinite(days) ? new Date(created_at) > new Date(Date.now() - days * 24 * 60 * 60 * 1000) : true)
                 .flatMap(({created_at: created, payload}) => Promise.all(payload.commits.map(async commit => ({created: new Date(created), ...(await rest.request(commit.url)).data})))),

source/plugins/contributors/index.mjs

@@ -51,7 +51,7 @@ export default async function({login, q, imports, data, rest, graphql, queries,
     //Compute contributors and contributions
     let contributors = {}
     for (const {author: {login, avatar_url: avatar}, commit: {message = "", author: {email = ""} = {}}} of commits) {
-      if ((!login) || (ignored.includes(login)) || (ignored.includes(email))) {
+      if ((!login) || (!imports.filters.text(login, ignored)) || (!imports.filters.text(email, ignored))) {
         console.debug(`metrics/compute/${login}/plugins > contributors > ignored contributor "${login}"`)
         continue
       }

source/plugins/habits/index.mjs

@@ -37,7 +37,7 @@ export default async function({login, data, rest, imports, q, account}, {enabled
     const commits = events
       .filter(({type}) => type === "PushEvent")
       .filter(({actor}) => account === "organization" ? true : actor.login?.toLocaleLowerCase() === login.toLocaleLowerCase())
-      .filter(({repo: {name: repo}}) => !((skipped.includes(repo.split("/").pop())) || (skipped.includes(repo))))
+      .filter(({repo: {name: repo}}) => imports.filters.repo(repo, skipped))
       .filter(({created_at}) => new Date(created_at) > new Date(Date.now() - days * 24 * 60 * 60 * 1000))
     console.debug(`metrics/compute/${login}/plugins > habits > filtered out ${commits.length} push events over last ${days} days`)
     habits.commits.fetched = commits.length

source/plugins/languages/analyzers.mjs

@@ -48,10 +48,8 @@ export async function indepth({login, data, imports, repositories, gpg}, {skippe
         break
 
       //Skip repository if asked
-      if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) {
+      if (!imports.filters.repo(repository, skipped))
-        console.debug(`metrics/compute/${login}/plugins > languages > skipped repository ${repository.owner.login}/${repository.name}`)
         continue
-      }
 
       //Repository handle
       const repo = `${repository.owner.login}/${repository.name}`
@@ -112,7 +110,7 @@ export async function recent({login, data, imports, rest, account}, {skipped = [
           ...(await rest.activity.listEventsForAuthenticatedUser({username: login, per_page: 100, page})).data
             .filter(({type}) => type === "PushEvent")
             .filter(({actor}) => account === "organization" ? true : actor.login?.toLocaleLowerCase() === login.toLocaleLowerCase())
-            .filter(({repo: {name: repo}}) => (!skipped.includes(repo.toLocaleLowerCase())) && (!skipped.includes(repo.toLocaleLowerCase().split("/").pop())))
+            .filter(({repo: {name: repo}}) => imports.filters.repo(repo, skipped))
             .filter(({created_at}) => new Date(created_at) > new Date(Date.now() - days * 24 * 60 * 60 * 1000)),
         )
       }

source/plugins/languages/index.mjs

@@ -48,10 +48,8 @@ export default async function({login, data, imports, q, rest, account}, {enabled
     const customColors = {}
     for (const repository of data.user.repositories.nodes) {
       //Skip repository if asked
-      if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) {
+      if (!imports.filters.repo(repository, skipped))
-        console.debug(`metrics/compute/${login}/plugins > languages > skipped repository ${repository.owner.login}/${repository.name}`)
         continue
-      }
       //Process repository languages
       for (const {size, node: {color, name}} of Object.values(repository.languages.edges)) {
         languages.stats[name] = (languages.stats[name] ?? 0) + size
@@ -125,7 +123,7 @@ export default async function({login, data, imports, q, rest, account}, {enabled
     //Compute languages stats
     for (const {section, stats = {}, lines = {}, missed = {bytes: 0}, total = 0} of [{section: "favorites", stats: languages.stats, lines: languages.lines, total: languages.total, missed: languages.missed}, {section: "recent", ...languages["stats.recent"]}]) {
       console.debug(`metrics/compute/${login}/plugins > languages > computing stats ${section}`)
-      languages[section] = Object.entries(stats).filter(([name]) => !ignored.includes(name.toLocaleLowerCase())).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size: value, color: languages.colors[name], x: 0})).filter(({value}) => value / total > threshold)
+      languages[section] = Object.entries(stats).filter(([name]) => imports.filters.text(name, ignored)).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size: value, color: languages.colors[name], x: 0})).filter(({value}) => value / total > threshold)
       if (other) {
         let value = indepth ? missed.bytes : Object.entries(stats).filter(([name]) => !Object.values(languages[section]).map(({name}) => name).includes(name)).reduce((a, [_, b]) => a + b, 0)
         if (value) {

source/plugins/lines/index.mjs

@@ -23,7 +23,7 @@ export default async function({login, data, imports, rest, q, account}, {enabled
     //Get contributors stats from repositories
     console.debug(`metrics/compute/${login}/plugins > lines > querying api`)
     const repos = {}, weeks = {}
-    const response = [...await Promise.allSettled(repositories.map(async ({repo, owner}) => (skipped.includes(repo.toLocaleLowerCase())) || (skipped.includes(`${owner}/${repo}`.toLocaleLowerCase())) ? {} : {handle: `${owner}/${repo}`, stats: (await rest.repos.getContributorsStats({owner, repo})).data}))].filter(({status}) => status === "fulfilled").map((
+    const response = [...await Promise.allSettled(repositories.map(async ({repo, owner}) => imports.filters.repo(`${owner}/${repo}`, skipped) ? {handle: `${owner}/${repo}`, stats: (await rest.repos.getContributorsStats({owner, repo})).data} : {}))].filter(({status}) => status === "fulfilled").map((
       {value},
     ) => value)
 

source/plugins/notable/index.mjs

@@ -20,9 +20,9 @@ export default async function({login, q, imports, rest, graphql, data, account,
         const {user: {repositoriesContributedTo: {edges}}} = await graphql(queries.notable.contributions({login, types: types.map(x => x.toLocaleUpperCase()).join(", "), after: cursor ? `after: "${cursor}"` : "", self, repositories: data.shared["repositories.batch"] || 100}))
         cursor = edges?.[edges?.length - 1]?.cursor
         edges
-          .filter(({node}) => !((skipped.includes(node.nameWithOwner.toLocaleLowerCase())) || (skipped.includes(node.nameWithOwner.split("/")[1].toLocaleLowerCase()))))
+          .filter(({node}) => imports.filters.repo(node, skipped))
           .filter(({node}) => ({all: true, organization: node.isInOrganization, user: !node.isInOrganization}[from]))
-          .filter(({node}) => imports.ghfilter(filter, {name: node.nameWithOwner, user: node.owner.login, stars: node.stargazers.totalCount, watchers: node.watchers.totalCount, forks: node.forks.totalCount}))
+          .filter(({node}) => imports.filters.github(filter, {name: node.nameWithOwner, user: node.owner.login, stars: node.stargazers.totalCount, watchers: node.watchers.totalCount, forks: node.forks.totalCount}))
           .map(({node}) => contributions.push({handle: node.nameWithOwner, stars: node.stargazers.totalCount, issues: node.issues.totalCount, pulls: node.pullRequests.totalCount, organization: node.isInOrganization, avatarUrl: node.owner.avatarUrl}))
         pushed = edges.length
       } while ((pushed) && (cursor))
@@ -129,7 +129,7 @@ export default async function({login, q, imports, rest, graphql, data, account,
       //Normalize contribution percentage
       contributions.map(aggregate => aggregate.user ? aggregate.user.percentage /= aggregate.aggregated : null)
       //Additional filtering (no user commits means that API wasn't able to answer back, considering it as matching by default)
-      contributions = contributions.filter(({handle, user}) => !user?.commits ? true : imports.ghfilter(filter, {handle, commits: contributions.history, "commits.user": user.commits, "commits.user%": user.percentage * 100, maintainer: user.maintainer}))
+      contributions = contributions.filter(({handle, user}) => !user?.commits ? true : imports.filters.github(filter, {handle, commits: contributions.history, "commits.user": user.commits, "commits.user%": user.percentage * 100, maintainer: user.maintainer}))
       //Sort contribution by maintainer first and then by contribution percentage
       contributions = contributions.sort((a, b) => ((b.user?.percentage + b.user?.maintainer) || 0) - ((a.user?.percentage + a.user?.maintainer) || 0))
     }

source/plugins/reactions/index.mjs

@@ -23,7 +23,7 @@ export default async function({login, q, imports, data, graphql, queries, accoun
           cursor = edges?.[0]?.cursor
           //Save issue comments
           const filtered = edges
-            .flatMap(({node: {createdAt: created, reactions: {nodes: reactions}}}) => ({created: new Date(created), reactions: reactions.filter(({user = {}}) => !ignored.includes(user.login)).map(({content}) => content)}))
+            .flatMap(({node: {createdAt: created, reactions: {nodes: reactions}}}) => ({created: new Date(created), reactions: reactions.filter(({user = {}}) => imports.filters.text(user.login, ignored)).map(({content}) => content)}))
             .filter(comment => Number.isFinite(days) ? comment.created < new Date(Date.now() - days * 24 * 60 * 60 * 1000) : true)
           pushed = filtered.length
           fetched.push(...filtered)

source/plugins/traffic/index.mjs

@@ -16,7 +16,7 @@ export default async function({login, imports, data, rest, q, account}, {enabled
     //Get views stats from repositories
     console.debug(`metrics/compute/${login}/plugins > traffic > querying api`)
     const views = {count: 0, uniques: 0}
-    const response = [...await Promise.allSettled(repositories.map(({repo, owner}) => (skipped.includes(repo.toLocaleLowerCase())) || (skipped.includes(`${owner}/${repo}`.toLocaleLowerCase())) ? {} : rest.repos.getViews({owner, repo})))].filter(({status}) => status === "fulfilled").map(({value}) => value)
+    const response = [...await Promise.allSettled(repositories.map(({repo, owner}) => imports.filters.repo(`${owner}/${repo}`, skipped) ? rest.repos.getViews({owner, repo}) : {}))].filter(({status}) => status === "fulfilled").map(({value}) => value)
 
     //Compute views
     console.debug(`metrics/compute/${login}/plugins > traffic > computing stats`)

source/plugins/wakatime/index.mjs

@@ -37,7 +37,7 @@ export default async function({login, q, imports, data, account}, {enabled = fal
         total: (others ? stats.total_seconds_including_other_language : stats.total_seconds) / (60 * 60),
         daily: (others ? stats.daily_average_including_other_language : stats.daily_average) / (60 * 60),
       },
-      languages: stats.languages?.map(({name, percent, total_seconds: total}) => ({name, percent: percent / 100, total})).filter(({name}) => _ignored.length ? !_ignored.includes(name.toLocaleLowerCase()) : true).sort((a, b) => b.percent - a.percent).slice(0, limit),
+      languages: stats.languages?.map(({name, percent, total_seconds: total}) => ({name, percent: percent / 100, total})).filter(({name}) => imports.filters.text(name, _ignored)).sort((a, b) => b.percent - a.percent).slice(0, limit),
       os: stats.operating_systems?.map(({name, percent, total_seconds: total}) => ({name, percent: percent / 100, total})).sort((a, b) => b.percent - a.percent).slice(0, limit),
       editors: stats.editors?.map(({name, percent, total_seconds: total}) => ({name, percent: percent / 100, total})).sort((a, b) => b.percent - a.percent).slice(0, limit),
     }