ref(plugins): centralize skip and ignore filters (#1238)

2022-09-21 23:50:18 -04:00
parent b0244de50e
commit 8523db79cb
13 changed files with 106 additions and 71 deletions
--- a/source/app/metrics/utils.mjs
+++ b/source/app/metrics/utils.mjs
@@ -323,8 +323,10 @@ export async function markdown(text, {mode = "inline", codelines = Infinity} = {
  return rendered
 }

-/**Check GitHub filter against object */
-export function ghfilter(text, object) {
+/**Filters */
+export const filters = {
+  /**GitHub query filter */
+  github(text, object) {
    console.debug(`metrics/svg/ghquery > checking ${text} against ${JSON.stringify(object)}`)
    const result = text.split(/(?<!NOT) /).map(x => x.trim()).filter(x => x).map(criteria => {
      const [key, filters] = criteria.split(":")
@@ -368,6 +370,43 @@ export function ghfilter(text, object) {
    }).reduce((a, b) => a && b, true)
    console.debug(`metrics/svg/ghquery > ${result ? "matching" : "not matching"}`)
    return result
+  },
+  /**Repository filter*/
+  repo(repository, patterns) {
+    //Disable filtering when no pattern is provided
+    if (!patterns.length)
+      return true
+
+    //Normalize repository handle
+    let repo, user
+    if (repository.nameWithOwner)
+      repository = repository.nameWithOwner
+    if ((repository.name)&&(repository.owner?.login)) {
+      user = repository.owner.login
+      repo = repository.name
+    }
+    user = (user ?? repository.split("/")[0]).toLocaleLowerCase()
+    repo = (repo ?? repository.split("/")[1]).toLocaleLowerCase()
+
+    //Basic pattern matching
+    const include = (!patterns.includes(repo)) && (!patterns.includes(`${user}/${repo}`))
+    console.debug(`metrics/filters/repo > filter ${repo} (${include ? "included" : "excluded"})`)
+    return include
+  },
+  /**Text filter*/
+  text(text, patterns) {
+    //Disable filtering when no pattern is provided
+    if (!patterns.length)
+      return true
+
+    //Normalize text
+    text = `${text}`.toLocaleLowerCase()
+
+    //Basic pattern matching
+    const include = !patterns.includes(text)
+    console.debug(`metrics/filters/text > filter ${text} (${include ? "included" : "excluded"})`)
+    return include
+  }
 }

 /**Image to base64 */
--- a/source/plugins/achievements/index.mjs
+++ b/source/plugins/achievements/index.mjs
@@ -23,7 +23,7 @@ export default async function({login, q, imports, data, computed, graphql, queri
    const achievements = list
      .filter(a => (order[a.rank] >= order[threshold]) || ((a.rank === "$") && (secrets)))
      .filter(a => (!only.length) || ((only.length) && (only.includes(a.title.toLocaleLowerCase()))))
-      .filter(a => !ignored.includes(a.title.toLocaleLowerCase()))
+      .filter(a => imports.filters.text(a.title, ignored))
      .sort((a, b) => (order[b.rank] + b.progress * 0.99) - (order[a.rank] + a.progress * 0.99))
      .map(({title, unlock, ...achievement}) => ({
        prefix: ({S: "Master", A: "Super", B: "Great"}[achievement.rank] ?? ""),
--- a/source/plugins/activity/index.mjs
+++ b/source/plugins/activity/index.mjs
@@ -46,7 +46,7 @@ export default async function({login, data, rest, q, account, imports}, {enabled
        .map(async ({type, payload, actor: {login: actor}, repo: {name: repo}, created_at}) => {
          //See https://docs.github.com/en/free-pro-team@latest/developers/webhooks-and-events/github-event-types
          const timestamp = new Date(created_at)
-          if ((skipped.includes(repo.split("/").pop())) || (skipped.includes(repo)))
+          if (!imports.filters.repo(repo, skipped))
            return null
          switch (type) {
            //Commented on a commit
@@ -54,7 +54,7 @@ export default async function({login, data, rest, q, account, imports}, {enabled
              if (!["created"].includes(payload.action))
                return null
              const {comment: {user: {login: user}, commit_id: sha, body: content}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                return null
              return {type: "comment", on: "commit", actor, timestamp, repo, content: await imports.markdown(content, {mode: markdown, codelines}), user, mobile: null, number: sha.substring(0, 7), title: ""}
            }
@@ -83,7 +83,7 @@ export default async function({login, data, rest, q, account, imports}, {enabled
              if (!["created"].includes(payload.action))
                return null
              const {issue: {user: {login: user}, title, number}, comment: {body: content, performed_via_github_app: mobile}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                return null
              return {type: "comment", on: "issue", actor, timestamp, repo, content: await imports.markdown(content, {mode: markdown, codelines}), user, mobile, number, title}
            }
@@ -92,7 +92,7 @@ export default async function({login, data, rest, q, account, imports}, {enabled
              if (!["opened", "closed", "reopened"].includes(payload.action))
                return null
              const {action, issue: {user: {login: user}, title, number, body: content}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                return null
              return {type: "issue", actor, timestamp, repo, action, user, number, title, content: await imports.markdown(content, {mode: markdown, codelines})}
            }
@@ -101,7 +101,7 @@ export default async function({login, data, rest, q, account, imports}, {enabled
              if (!["added"].includes(payload.action))
                return null
              const {member: {login: user}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                return null
              return {type: "member", actor, timestamp, repo, user}
            }
@@ -114,14 +114,14 @@ export default async function({login, data, rest, q, account, imports}, {enabled
              if (!["opened", "closed"].includes(payload.action))
                return null
              const {action, pull_request: {user: {login: user}, title, number, body: content, additions: added, deletions: deleted, changed_files: changed, merged}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                return null
              return {type: "pr", actor, timestamp, repo, action: (action === "closed") && (merged) ? "merged" : action, user, title, number, content: await imports.markdown(content, {mode: markdown, codelines}), lines: {added, deleted}, files: {changed}}
            }
            //Reviewed a pull request
            case "PullRequestReviewEvent": {
              const {review: {state: review}, pull_request: {user: {login: user}, number, title}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                return null
              return {type: "review", actor, timestamp, repo, review, user, number, title}
            }
@@ -130,14 +130,14 @@ export default async function({login, data, rest, q, account, imports}, {enabled
              if (!["created"].includes(payload.action))
                return null
              const {pull_request: {user: {login: user}, title, number}, comment: {body: content, performed_via_github_app: mobile}} = payload
-              if (ignored.includes(user))
+              if (!imports.filters.text(user, ignored))
                return null
              return {type: "comment", on: "pr", actor, timestamp, repo, content: await imports.markdown(content, {mode: markdown, codelines}), user, mobile, number, title}
            }
            //Pushed commits
            case "PushEvent": {
              let {size, commits, ref} = payload
-              commits = commits.filter(({author: {email}}) => !ignored.includes(email))
+              commits = commits.filter(({author: {email}}) => imports.filters.text(email, ignored))
              if (!commits.length)
                return null
              if (commits.slice(-1).pop()?.message.startsWith("Merge branch "))
--- a/source/plugins/code/index.mjs
+++ b/source/plugins/code/index.mjs
@@ -35,7 +35,7 @@ export default async function({login, q, imports, data, rest, account}, {enabled
                : await rest.activity.listEventsForAuthenticatedUser({username: login, per_page: 100, page})).data
                .filter(({type}) => type === "PushEvent")
                .filter(({actor}) => account === "organization" ? true : actor.login?.toLocaleLowerCase() === login.toLocaleLowerCase())
-                .filter(({repo: {name: repo}}) => !((skipped.includes(repo.split("/").pop())) || (skipped.includes(repo))))
+                .filter(({repo: {name: repo}}) => imports.filters.repo(repo, skipped))
                .filter(event => visibility === "public" ? event.public : true)
                .filter(({created_at}) => Number.isFinite(days) ? new Date(created_at) > new Date(Date.now() - days * 24 * 60 * 60 * 1000) : true)
                .flatMap(({created_at: created, payload}) => Promise.all(payload.commits.map(async commit => ({created: new Date(created), ...(await rest.request(commit.url)).data})))),
--- a/source/plugins/contributors/index.mjs
+++ b/source/plugins/contributors/index.mjs
@@ -51,7 +51,7 @@ export default async function({login, q, imports, data, rest, graphql, queries,
    //Compute contributors and contributions
    let contributors = {}
    for (const {author: {login, avatar_url: avatar}, commit: {message = "", author: {email = ""} = {}}} of commits) {
-      if ((!login) || (ignored.includes(login)) || (ignored.includes(email))) {
+      if ((!login) || (!imports.filters.text(login, ignored)) || (!imports.filters.text(email, ignored))) {
        console.debug(`metrics/compute/${login}/plugins > contributors > ignored contributor "${login}"`)
        continue
      }
--- a/source/plugins/habits/index.mjs
+++ b/source/plugins/habits/index.mjs
@@ -37,7 +37,7 @@ export default async function({login, data, rest, imports, q, account}, {enabled
    const commits = events
      .filter(({type}) => type === "PushEvent")
      .filter(({actor}) => account === "organization" ? true : actor.login?.toLocaleLowerCase() === login.toLocaleLowerCase())
-      .filter(({repo: {name: repo}}) => !((skipped.includes(repo.split("/").pop())) || (skipped.includes(repo))))
+      .filter(({repo: {name: repo}}) => imports.filters.repo(repo, skipped))
      .filter(({created_at}) => new Date(created_at) > new Date(Date.now() - days * 24 * 60 * 60 * 1000))
    console.debug(`metrics/compute/${login}/plugins > habits > filtered out ${commits.length} push events over last ${days} days`)
    habits.commits.fetched = commits.length
--- a/source/plugins/languages/analyzers.mjs
+++ b/source/plugins/languages/analyzers.mjs
@@ -48,10 +48,8 @@ export async function indepth({login, data, imports, repositories, gpg}, {skippe
        break

      //Skip repository if asked
-      if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) {
-        console.debug(`metrics/compute/${login}/plugins > languages > skipped repository ${repository.owner.login}/${repository.name}`)
+      if (!imports.filters.repo(repository, skipped))
        continue
-      }

      //Repository handle
      const repo = `${repository.owner.login}/${repository.name}`
@@ -112,7 +110,7 @@ export async function recent({login, data, imports, rest, account}, {skipped = [
          ...(await rest.activity.listEventsForAuthenticatedUser({username: login, per_page: 100, page})).data
            .filter(({type}) => type === "PushEvent")
            .filter(({actor}) => account === "organization" ? true : actor.login?.toLocaleLowerCase() === login.toLocaleLowerCase())
-            .filter(({repo: {name: repo}}) => (!skipped.includes(repo.toLocaleLowerCase())) && (!skipped.includes(repo.toLocaleLowerCase().split("/").pop())))
+            .filter(({repo: {name: repo}}) => imports.filters.repo(repo, skipped))
            .filter(({created_at}) => new Date(created_at) > new Date(Date.now() - days * 24 * 60 * 60 * 1000)),
        )
      }
--- a/source/plugins/languages/index.mjs
+++ b/source/plugins/languages/index.mjs
@@ -48,10 +48,8 @@ export default async function({login, data, imports, q, rest, account}, {enabled
    const customColors = {}
    for (const repository of data.user.repositories.nodes) {
      //Skip repository if asked
-      if ((skipped.includes(repository.name.toLocaleLowerCase())) || (skipped.includes(`${repository.owner.login}/${repository.name}`.toLocaleLowerCase()))) {
-        console.debug(`metrics/compute/${login}/plugins > languages > skipped repository ${repository.owner.login}/${repository.name}`)
+      if (!imports.filters.repo(repository, skipped))
        continue
-      }
      //Process repository languages
      for (const {size, node: {color, name}} of Object.values(repository.languages.edges)) {
        languages.stats[name] = (languages.stats[name] ?? 0) + size
@@ -125,7 +123,7 @@ export default async function({login, data, imports, q, rest, account}, {enabled
    //Compute languages stats
    for (const {section, stats = {}, lines = {}, missed = {bytes: 0}, total = 0} of [{section: "favorites", stats: languages.stats, lines: languages.lines, total: languages.total, missed: languages.missed}, {section: "recent", ...languages["stats.recent"]}]) {
      console.debug(`metrics/compute/${login}/plugins > languages > computing stats ${section}`)
-      languages[section] = Object.entries(stats).filter(([name]) => !ignored.includes(name.toLocaleLowerCase())).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size: value, color: languages.colors[name], x: 0})).filter(({value}) => value / total > threshold)
+      languages[section] = Object.entries(stats).filter(([name]) => imports.filters.text(name, ignored)).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size: value, color: languages.colors[name], x: 0})).filter(({value}) => value / total > threshold)
      if (other) {
        let value = indepth ? missed.bytes : Object.entries(stats).filter(([name]) => !Object.values(languages[section]).map(({name}) => name).includes(name)).reduce((a, [_, b]) => a + b, 0)
        if (value) {
--- a/source/plugins/lines/index.mjs
+++ b/source/plugins/lines/index.mjs
@@ -23,7 +23,7 @@ export default async function({login, data, imports, rest, q, account}, {enabled
    //Get contributors stats from repositories
    console.debug(`metrics/compute/${login}/plugins > lines > querying api`)
    const repos = {}, weeks = {}
-    const response = [...await Promise.allSettled(repositories.map(async ({repo, owner}) => (skipped.includes(repo.toLocaleLowerCase())) || (skipped.includes(`${owner}/${repo}`.toLocaleLowerCase())) ? {} : {handle: `${owner}/${repo}`, stats: (await rest.repos.getContributorsStats({owner, repo})).data}))].filter(({status}) => status === "fulfilled").map((
+    const response = [...await Promise.allSettled(repositories.map(async ({repo, owner}) => imports.filters.repo(`${owner}/${repo}`, skipped) ? {handle: `${owner}/${repo}`, stats: (await rest.repos.getContributorsStats({owner, repo})).data} : {}))].filter(({status}) => status === "fulfilled").map((
      {value},
    ) => value)

--- a/source/plugins/notable/index.mjs
+++ b/source/plugins/notable/index.mjs
@@ -20,9 +20,9 @@ export default async function({login, q, imports, rest, graphql, data, account,
        const {user: {repositoriesContributedTo: {edges}}} = await graphql(queries.notable.contributions({login, types: types.map(x => x.toLocaleUpperCase()).join(", "), after: cursor ? `after: "${cursor}"` : "", self, repositories: data.shared["repositories.batch"] || 100}))
        cursor = edges?.[edges?.length - 1]?.cursor
        edges
-          .filter(({node}) => !((skipped.includes(node.nameWithOwner.toLocaleLowerCase())) || (skipped.includes(node.nameWithOwner.split("/")[1].toLocaleLowerCase()))))
+          .filter(({node}) => imports.filters.repo(node, skipped))
          .filter(({node}) => ({all: true, organization: node.isInOrganization, user: !node.isInOrganization}[from]))
-          .filter(({node}) => imports.ghfilter(filter, {name: node.nameWithOwner, user: node.owner.login, stars: node.stargazers.totalCount, watchers: node.watchers.totalCount, forks: node.forks.totalCount}))
+          .filter(({node}) => imports.filters.github(filter, {name: node.nameWithOwner, user: node.owner.login, stars: node.stargazers.totalCount, watchers: node.watchers.totalCount, forks: node.forks.totalCount}))
          .map(({node}) => contributions.push({handle: node.nameWithOwner, stars: node.stargazers.totalCount, issues: node.issues.totalCount, pulls: node.pullRequests.totalCount, organization: node.isInOrganization, avatarUrl: node.owner.avatarUrl}))
        pushed = edges.length
      } while ((pushed) && (cursor))
@@ -129,7 +129,7 @@ export default async function({login, q, imports, rest, graphql, data, account,
      //Normalize contribution percentage
      contributions.map(aggregate => aggregate.user ? aggregate.user.percentage /= aggregate.aggregated : null)
      //Additional filtering (no user commits means that API wasn't able to answer back, considering it as matching by default)
-      contributions = contributions.filter(({handle, user}) => !user?.commits ? true : imports.ghfilter(filter, {handle, commits: contributions.history, "commits.user": user.commits, "commits.user%": user.percentage * 100, maintainer: user.maintainer}))
+      contributions = contributions.filter(({handle, user}) => !user?.commits ? true : imports.filters.github(filter, {handle, commits: contributions.history, "commits.user": user.commits, "commits.user%": user.percentage * 100, maintainer: user.maintainer}))
      //Sort contribution by maintainer first and then by contribution percentage
      contributions = contributions.sort((a, b) => ((b.user?.percentage + b.user?.maintainer) || 0) - ((a.user?.percentage + a.user?.maintainer) || 0))
    }
--- a/source/plugins/reactions/index.mjs
+++ b/source/plugins/reactions/index.mjs
@@ -23,7 +23,7 @@ export default async function({login, q, imports, data, graphql, queries, accoun
          cursor = edges?.[0]?.cursor
          //Save issue comments
          const filtered = edges
-            .flatMap(({node: {createdAt: created, reactions: {nodes: reactions}}}) => ({created: new Date(created), reactions: reactions.filter(({user = {}}) => !ignored.includes(user.login)).map(({content}) => content)}))
+            .flatMap(({node: {createdAt: created, reactions: {nodes: reactions}}}) => ({created: new Date(created), reactions: reactions.filter(({user = {}}) => imports.filters.text(user.login, ignored)).map(({content}) => content)}))
            .filter(comment => Number.isFinite(days) ? comment.created < new Date(Date.now() - days * 24 * 60 * 60 * 1000) : true)
          pushed = filtered.length
          fetched.push(...filtered)
--- a/source/plugins/traffic/index.mjs
+++ b/source/plugins/traffic/index.mjs
@@ -16,7 +16,7 @@ export default async function({login, imports, data, rest, q, account}, {enabled
    //Get views stats from repositories
    console.debug(`metrics/compute/${login}/plugins > traffic > querying api`)
    const views = {count: 0, uniques: 0}
-    const response = [...await Promise.allSettled(repositories.map(({repo, owner}) => (skipped.includes(repo.toLocaleLowerCase())) || (skipped.includes(`${owner}/${repo}`.toLocaleLowerCase())) ? {} : rest.repos.getViews({owner, repo})))].filter(({status}) => status === "fulfilled").map(({value}) => value)
+    const response = [...await Promise.allSettled(repositories.map(({repo, owner}) => imports.filters.repo(`${owner}/${repo}`, skipped) ? rest.repos.getViews({owner, repo}) : {}))].filter(({status}) => status === "fulfilled").map(({value}) => value)

    //Compute views
    console.debug(`metrics/compute/${login}/plugins > traffic > computing stats`)
--- a/source/plugins/wakatime/index.mjs
+++ b/source/plugins/wakatime/index.mjs
@@ -37,7 +37,7 @@ export default async function({login, q, imports, data, account}, {enabled = fal
        total: (others ? stats.total_seconds_including_other_language : stats.total_seconds) / (60 * 60),
        daily: (others ? stats.daily_average_including_other_language : stats.daily_average) / (60 * 60),
      },
-      languages: stats.languages?.map(({name, percent, total_seconds: total}) => ({name, percent: percent / 100, total})).filter(({name}) => _ignored.length ? !_ignored.includes(name.toLocaleLowerCase()) : true).sort((a, b) => b.percent - a.percent).slice(0, limit),
+      languages: stats.languages?.map(({name, percent, total_seconds: total}) => ({name, percent: percent / 100, total})).filter(({name}) => imports.filters.text(name, _ignored)).sort((a, b) => b.percent - a.percent).slice(0, limit),
      os: stats.operating_systems?.map(({name, percent, total_seconds: total}) => ({name, percent: percent / 100, total})).sort((a, b) => b.percent - a.percent).slice(0, limit),
      editors: stats.editors?.map(({name, percent, total_seconds: total}) => ({name, percent: percent / 100, total})).sort((a, b) => b.percent - a.percent).slice(0, limit),
    }