feat(plugins/base): add bulk query and fallback to unit (#825)

source/plugins/base/index.mjs

@@ -30,53 +30,60 @@ export default async function({login, graphql, rest, data, q, queries, imports},
       const queried = await graphql(queries.base[account]({login}))
       Object.assign(data, {user:queried[account]})
       postprocess?.[account]({login, data})
-      //Query basic fields
+      try {
-      const fields = {
+        Object.assign(data.user, (await graphql(queries.base[`${account}.x`]({login, account, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString()})))[account])
-        user:["packages", "starredRepositories", "watching", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "followers", "following", "issueComments", "organizations", "repositoriesContributedTo(includeUserRepositories: true)"],
+        console.debug(`metrics/compute/${login}/base > successfully loaded bulk query`)
-        organization:["packages", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "membersWithRole"],
-      }[account] ?? []
-      for (const field of fields) {
-        try {
-          Object.assign(data.user, (await graphql(queries.base.field({login, account, field})))[account])
-        }
-        catch {
-          console.debug(`metrics/compute/${login}/base > failed to retrieve ${field}`)
-          data.user[field] = {totalCount:NaN}
-        }
       }
-      //Query repositories fields
+      catch {
-      for (const field of ["totalCount", "totalDiskUsage"]) {
+        console.debug(`metrics/compute/${login}/base > failed to load bulk query, falling back to unit queries`)
-        try {
+        //Query basic fields
-          Object.assign(data.user.repositories, (await graphql(queries.base["field.repositories"]({login, account, field})))[account].repositories)
+        const fields = {
-        }
+          user:["packages", "starredRepositories", "watching", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "followers", "following", "issueComments", "organizations", "repositoriesContributedTo(includeUserRepositories: true)"],
-        catch (error) {
+          organization:["packages", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "membersWithRole"],
-          console.log(error)
+        }[account] ?? []
-          console.debug(`metrics/compute/${login}/base > failed to retrieve repositories.${field}`)
+        for (const field of fields) {
-          data.user.repositories[field] = NaN
+          try {
-        }
+            Object.assign(data.user, (await graphql(queries.base.field({login, account, field})))[account])
-      }
+          }
-      //Query user account fields
+          catch {
-      if (account === "user") {
+            console.debug(`metrics/compute/${login}/base > failed to retrieve ${field}`)
-        //Query contributions collection
+            data.user[field] = {totalCount:NaN}
-        {
-          const fields = ["totalRepositoriesWithContributedCommits", "totalCommitContributions", "restrictedContributionsCount", "totalIssueContributions", "totalPullRequestContributions", "totalPullRequestReviewContributions"]
-          for (const field of fields) {
-            try {
-              Object.assign(data.user.contributionsCollection, (await graphql(queries.base.contributions({login, account, field})))[account].contributionsCollection)
-            }
-            catch {
-              console.debug(`metrics/compute/${login}/base > failed to retrieve contributionsCollection.${field}`)
-              data.user.contributionsCollection[field] = NaN
-            }
           }
         }
-        //Query calendar
+        //Query repositories fields
-        try {
+        for (const field of ["totalCount", "totalDiskUsage"]) {
-          Object.assign(data.user, (await graphql(queries.base.calendar({login, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString()})))[account])
+          try {
+            Object.assign(data.user.repositories, (await graphql(queries.base["field.repositories"]({login, account, field})))[account].repositories)
+          }
+          catch (error) {
+            console.log(error)
+            console.debug(`metrics/compute/${login}/base > failed to retrieve repositories.${field}`)
+            data.user.repositories[field] = NaN
+          }
         }
-        catch {
+        //Query user account fields
-          console.debug(`metrics/compute/${login}/base > failed to retrieve contributions calendar`)
+        if (account === "user") {
-          data.user.calendar = {contributionCalendar:{weeks:[]}}
+          //Query contributions collection
+          {
+            const fields = ["totalRepositoriesWithContributedCommits", "totalCommitContributions", "restrictedContributionsCount", "totalIssueContributions", "totalPullRequestContributions", "totalPullRequestReviewContributions"]
+            for (const field of fields) {
+              try {
+                Object.assign(data.user.contributionsCollection, (await graphql(queries.base.contributions({login, account, field})))[account].contributionsCollection)
+              }
+              catch {
+                console.debug(`metrics/compute/${login}/base > failed to retrieve contributionsCollection.${field}`)
+                data.user.contributionsCollection[field] = NaN
+              }
+            }
+          }
+          //Query calendar
+          try {
+            Object.assign(data.user, (await graphql(queries.base.calendar({login, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString()})))[account])
+          }
+          catch {
+            console.debug(`metrics/compute/${login}/base > failed to retrieve contributions calendar`)
+            data.user.calendar = {contributionCalendar:{weeks:[]}}
+          }
         }
       }
       //Query repositories from GitHub API
@@ -107,6 +114,10 @@ export default async function({login, graphql, rest, data, q, queries, imports},
           data.user[type].nodes.push(...nodes)
           pushed = nodes.length
           console.debug(`metrics/compute/${login}/base > retrieved ${pushed} ${type} after ${cursor}`)
+          if (pushed < repositories) {
+            console.debug(`metrics/compute/${login}/base > retrieved less repositories than expected, probably no more to fetch`)
+            break
+          }
         } while ((pushed) && (cursor) && ((data.user.repositories?.nodes?.length ?? 0) + (data.user.repositoriesContributedTo?.nodes?.length ?? 0) < repositories))
         //Limit repositories
         console.debug(`metrics/compute/${login}/base > keeping only ${repositories} ${type}`)

source/plugins/base/queries/organization.x.graphql

@@ -0,0 +1,20 @@
+query BaseOrganizationX {
+  organization(login: "$login") {
+    packages {
+      totalCount
+    }
+    sponsorshipsAsSponsor {
+      totalCount
+    }
+    sponsorshipsAsMaintainer {
+      totalCount
+    }
+    membersWithRole {
+      totalCount
+    }
+    repositories(last: 0) {
+      totalCount
+      totalDiskUsage
+    }
+  }
+}

source/plugins/base/queries/user.x.graphql

@@ -0,0 +1,55 @@
+query BaseUserX {
+  user(login: "$login") {
+    packages {
+      totalCount
+    }
+    starredRepositories {
+      totalCount
+    }
+    watching {
+      totalCount
+    }
+    sponsorshipsAsSponsor {
+      totalCount
+    }
+    sponsorshipsAsMaintainer {
+      totalCount
+    }
+    followers {
+      totalCount
+    }
+    following {
+      totalCount
+    }
+    issueComments {
+      totalCount
+    }
+    organizations {
+      totalCount
+    }
+    repositoriesContributedTo(includeUserRepositories: true) {
+      totalCount
+    }
+    repositories(last: 0) {
+      totalCount
+      totalDiskUsage
+    }
+    contributionsCollection {
+      totalRepositoriesWithContributedCommits
+      totalCommitContributions
+      restrictedContributionsCount
+      totalIssueContributions
+      totalPullRequestContributions
+      totalPullRequestReviewContributions
+    }
+    calendar:contributionsCollection(from: "$calendar.from", to: "$calendar.to") {
+      contributionCalendar {
+        weeks {
+          contributionDays {
+            color
+          }
+        }
+      }
+    }
+  }
+}