feat(plugins/base): split base queries to reduce GitHub API timeouts (#584)

source/app/mocks/api/github/graphql/base.calendar.mjs

@@ -0,0 +1,41 @@
+/**Mocked data */
+export default function({faker, query, login = faker.internet.userName()}) {
+  console.debug("metrics/compute/mocks > mocking graphql api result > base/user")
+  return ({
+    user:{
+      calendar:{
+        contributionCalendar:{
+          weeks:[
+            {
+              contributionDays:[
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+              ],
+            },
+            {
+              contributionDays:[
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+              ],
+            },
+            {
+              contributionDays:[
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
+              ],
+            },
+          ],
+        },
+      },
+    },
+  })
+}

source/app/mocks/api/github/graphql/base.contributions.mjs

@@ -0,0 +1,16 @@
+/**Mocked data */
+export default function({faker, query, login = faker.internet.userName()}) {
+  console.debug("metrics/compute/mocks > mocking graphql api result > base/user")
+  return ({
+    user:{
+      contributionsCollection:{
+        totalRepositoriesWithContributedCommits:faker.datatype.number(100),
+        totalCommitContributions:faker.datatype.number(10000),
+        restrictedContributionsCount:faker.datatype.number(10000),
+        totalIssueContributions:faker.datatype.number(100),
+        totalPullRequestContributions:faker.datatype.number(1000),
+        totalPullRequestReviewContributions:faker.datatype.number(1000),
+      },
+    },
+  })
+}

source/app/mocks/api/github/graphql/base.field.mjs

@@ -0,0 +1,18 @@
+/**Mocked data */
+export default function({faker, query, login = faker.internet.userName()}) {
+  console.debug("metrics/compute/mocks > mocking graphql api result > base/user")
+  return ({
+    user:{
+      packages:{totalCount:faker.datatype.number(10)},
+      starredRepositories:{totalCount:faker.datatype.number(1000)},
+      watching:{totalCount:faker.datatype.number(100)},
+      sponsorshipsAsSponsor:{totalCount:faker.datatype.number(10)},
+      sponsorshipsAsMaintainer:{totalCount:faker.datatype.number(10)},
+      repositoriesContributedTo:{totalCount:faker.datatype.number(100)},
+      followers:{totalCount:faker.datatype.number(1000)},
+      following:{totalCount:faker.datatype.number(1000)},
+      issueComments:{totalCount:faker.datatype.number(1000)},
+      organizations:{totalCount:faker.datatype.number(10)},
+    },
+  })
+}

source/app/mocks/api/github/graphql/base.field.repositories.mjs

@@ -0,0 +1,9 @@
+/**Mocked data */
+export default function({faker, query, login = faker.internet.userName()}) {
+  console.debug("metrics/compute/mocks > mocking graphql api result > base/user")
+  return ({
+    user:{
+      repositories:{totalCount:faker.datatype.number(100), totalDiskUsage:faker.datatype.number(100000)},
+    },
+  })
+}

source/app/mocks/api/github/graphql/base.user.mjs

@@ -11,58 +11,6 @@ export default function({faker, query, login = faker.internet.userName()}) {
       websiteUrl:faker.internet.url(),
       isHireable:faker.datatype.boolean(),
       twitterUsername:login,
-      repositories:{totalCount:faker.datatype.number(100), totalDiskUsage:faker.datatype.number(100000)},
-      packages:{totalCount:faker.datatype.number(10)},
-      starredRepositories:{totalCount:faker.datatype.number(1000)},
-      watching:{totalCount:faker.datatype.number(100)},
-      sponsorshipsAsSponsor:{totalCount:faker.datatype.number(10)},
-      sponsorshipsAsMaintainer:{totalCount:faker.datatype.number(10)},
-      contributionsCollection:{
-        totalRepositoriesWithContributedCommits:faker.datatype.number(100),
-        totalCommitContributions:faker.datatype.number(10000),
-        restrictedContributionsCount:faker.datatype.number(10000),
-        totalIssueContributions:faker.datatype.number(100),
-        totalPullRequestContributions:faker.datatype.number(1000),
-        totalPullRequestReviewContributions:faker.datatype.number(1000),
-      },
-      calendar:{
-        contributionCalendar:{
-          weeks:[
-            {
-              contributionDays:[
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-              ],
-            },
-            {
-              contributionDays:[
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-              ],
-            },
-            {
-              contributionDays:[
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-                {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
-              ],
-            },
-          ],
-        },
-      },
-      repositoriesContributedTo:{totalCount:faker.datatype.number(100)},
-      followers:{totalCount:faker.datatype.number(1000)},
-      following:{totalCount:faker.datatype.number(1000)},
-      issueComments:{totalCount:faker.datatype.number(1000)},
-      organizations:{totalCount:faker.datatype.number(10)},
     },
   })
 }

source/plugins/base/index.mjs

@@ -27,16 +27,65 @@ export default async function({login, graphql, rest, data, q, queries, imports},
     try {
       //Query data from GitHub API
       console.debug(`metrics/compute/${login}/base > account ${account}`)
-      const queried = await graphql(queries.base[account]({login, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString(), forks, affiliations}))
+      const queried = await graphql(queries.base[account]({login}))
       Object.assign(data, {user:queried[account]})
       postprocess?.[account]({login, data})
+      //Query basic fields
+      const fields = {
+        user:["packages", "starredRepositories", "watching", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "followers", "following", "issueComments", "organizations", "repositoriesContributedTo(includeUserRepositories: true)"],
+        organization:["packages", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "membersWithRole"],
+      }[account] ?? []
+      for (const field of fields) {
+        try {
+          Object.assign(data.user, (await graphql(queries.base.field({login, account, field})))[account])
+        }
+        catch {
+          console.debug(`metrics/compute/${login}/base > failed to retrieve ${field}`)
+          data.user[field] = {totalCount:NaN}
+        }
+      }
+      //Query repositories fields
+      for (const field of ["totalCount", "totalDiskUsage"]) {
+        try {
+          Object.assign(data.user.repositories, (await graphql(queries.base["field.repositories"]({login, account, field})))[account].repositories)
+        }
+        catch (error) {
+          console.log(error)
+          console.debug(`metrics/compute/${login}/base > failed to retrieve repositories.${field}`)
+          data.user.repositories[field] = NaN
+        }
+      }
+      //Query user account fields
+      if (account === "user") {
+        //Query contributions collection
+        {
+          const fields = ["totalRepositoriesWithContributedCommits", "totalCommitContributions", "restrictedContributionsCount", "totalIssueContributions", "totalPullRequestContributions", "totalPullRequestReviewContributions"]
+          for (const field of fields) {
+            try {
+              Object.assign(data.user.contributionsCollection, (await graphql(queries.base.contributions({login, account, field})))[account].contributionsCollection)
+            }
+            catch {
+              console.debug(`metrics/compute/${login}/base > failed to retrieve contributionsCollection.${field}`)
+              data.user.contributionsCollection[field] = NaN
+            }
+          }
+        }
+        //Query calendar
+        try {
+          Object.assign(data.user, (await graphql(queries.base.calendar({login, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString()})))[account])
+        }
+        catch {
+          console.debug(`metrics/compute/${login}/base > failed to retrieve contributions calendar`)
+          data.user.calendar = {contributionCalendar:{weeks:[]}}
+        }
+      }
       //Query repositories from GitHub API
-      data.user.repositoriesContributedTo.nodes = data.user.repositoriesContributedTo.nodes ?? []
       for (const type of ({user:["repositories", "repositoriesContributedTo"], organization:["repositories"]}[account] ?? [])) {
         //Iterate through repositories
         let cursor = null
         let pushed = 0
         const options = {repositories:{forks, affiliations, constraints:""}, repositoriesContributedTo:{forks:"", affiliations:"", constraints:", includeUserRepositories: false, contributionTypes: COMMIT"}}[type] ?? null
+        data.user[type] = data.user[type] ?? {}
         data.user[type].nodes = data.user[type].nodes ?? []
         do {
           console.debug(`metrics/compute/${login}/base > retrieving ${type} after ${cursor}`)
@@ -45,7 +94,7 @@ export default async function({login, graphql, rest, data, q, queries, imports},
           data.user[type].nodes.push(...nodes)
           pushed = nodes.length
           console.debug(`metrics/compute/${login}/base > retrieved ${pushed} ${type} after ${cursor}`)
-        } while ((pushed) && (cursor) && (data.user.repositories.nodes.length + data.user.repositoriesContributedTo.nodes.length < repositories))
+        } while ((pushed) && (cursor) && ((data.user.repositories?.nodes?.length ?? 0) + (data.user.repositoriesContributedTo?.nodes?.length ?? 0) < repositories))
         //Limit repositories
         console.debug(`metrics/compute/${login}/base > keeping only ${repositories} ${type}`)
         data.user[type].nodes.splice(repositories)
@@ -93,6 +142,8 @@ const postprocess = {
     data.account = "user"
     Object.assign(data.user, {
       isVerified:false,
+      repositories:{},
+      contributionsCollection:{},
     })
   },
   //Organization
@@ -101,22 +152,23 @@ const postprocess = {
     data.account = "organization"
     Object.assign(data.user, {
       isHireable:false,
-      starredRepositories:{totalCount:0},
+      repositories:{},
-      watching:{totalCount:0},
+      starredRepositories:{totalCount:NaN},
+      watching:{totalCount:NaN},
       contributionsCollection:{
-        totalRepositoriesWithContributedCommits:0,
+        totalRepositoriesWithContributedCommits:NaN,
-        totalCommitContributions:0,
+        totalCommitContributions:NaN,
-        restrictedContributionsCount:0,
+        restrictedContributionsCount:NaN,
-        totalIssueContributions:0,
+        totalIssueContributions:NaN,
-        totalPullRequestContributions:0,
+        totalPullRequestContributions:NaN,
-        totalPullRequestReviewContributions:0,
+        totalPullRequestReviewContributions:NaN,
       },
       calendar:{contributionCalendar:{weeks:[]}},
-      repositoriesContributedTo:{totalCount:0},
+      repositoriesContributedTo:{totalCount:NaN, nodes:[]},
-      followers:{totalCount:0},
+      followers:{totalCount:NaN},
-      following:{totalCount:0},
+      following:{totalCount:NaN},
-      issueComments:{totalCount:0},
+      issueComments:{totalCount:NaN},
-      organizations:{totalCount:0},
+      organizations:{totalCount:NaN},
     })
   },
   //Skip base content query and instantiate an empty user instance
@@ -127,16 +179,16 @@ const postprocess = {
       postprocess?.[account]({login, data})
     data.account = "bypass"
     Object.assign(data.user, {
-      databaseId:0,
+      databaseId:NaN,
       name:login,
       login,
       createdAt:new Date(),
       avatarUrl:`https://github.com/${login}.png`,
       websiteUrl:null,
       twitterUsername:login,
-      repositories:{totalCount:0, totalDiskUsage:0, nodes:[]},
+      repositories:{totalCount:NaN, totalDiskUsage:NaN, nodes:[]},
-      packages:{totalCount:0},
+      packages:{totalCount:NaN},
-      repositoriesContributedTo:{nodes:[]},
+      repositoriesContributedTo:{totalCount:NaN, nodes:[]},
     })
   },
 }

source/plugins/base/queries/calendar.graphql

@@ -0,0 +1,13 @@
+query BaseCalendar {
+  user(login: "$login") {
+    calendar:contributionsCollection(from: "$calendar.from", to: "$calendar.to") {
+      contributionCalendar {
+        weeks {
+          contributionDays {
+            color
+          }
+        }
+      }
+    }
+  }
+}

source/plugins/base/queries/contributions.graphql

@@ -0,0 +1,9 @@
+query BaseContributions {
+  user(login: "$login") {
+    contributionsCollection {
+      $field
+    }
+  }
+}
+
+

source/plugins/base/queries/field.graphql

@@ -0,0 +1,7 @@
+query BaseField {
+  $account(login: "$login") {
+    $field {
+      totalCount
+    }
+  }
+}

source/plugins/base/queries/field.repositories.graphql

@@ -0,0 +1,7 @@
+query BaseFieldRepositories {
+  $account(login: "$login") {
+    repositories(last: 0) {
+      $field
+    }
+  }
+}

source/plugins/base/queries/organization.graphql

@@ -9,21 +9,5 @@ query BaseOrganization {
     websiteUrl
     isVerified
     twitterUsername
-    repositories(last: 0) {
-      totalCount
-      totalDiskUsage
-    }
-    packages {
-      totalCount
-    }
-		sponsorshipsAsSponsor {
-      totalCount
-    }
-    sponsorshipsAsMaintainer {
-      totalCount
-    }
-    membersWithRole {
-      totalCount
-    }
   }
 }

source/plugins/base/queries/repositories.graphql

@@ -10,12 +10,22 @@ query BaseRepositories {
           login
         }
         isFork
+        forkCount
         watchers {
           totalCount
         }
         stargazers {
           totalCount
         }
+        releases {
+          totalCount
+        }
+        deployments {
+          totalCount
+        }
+        environments {
+          totalCount
+        }
         languages(first: 8) {
           edges {
             size
@@ -25,6 +35,10 @@ query BaseRepositories {
             }
           }
         }
+        licenseInfo {
+          name
+          spdxId
+        }
         issues_open: issues(states: OPEN) {
           totalCount
         }
@@ -40,20 +54,6 @@ query BaseRepositories {
         pr_merged: pullRequests(states: MERGED) {
           totalCount
         }
-        releases {
-          totalCount
-        }
-        forkCount
-        licenseInfo {
-          name
-          spdxId
-        }
-        deployments {
-          totalCount
-        }
-        environments {
-          totalCount
-        }
       }
     }
   }

source/plugins/base/queries/repository.graphql

@@ -2,19 +2,29 @@ query BaseRepository {
   $account(login: "$login") {
     repository(name: "$repo") {
       name
+      createdAt
+      diskUsage
+      homepageUrl
       owner {
         login
       }
       isFork
-      createdAt
+      forkCount
-      diskUsage
-      homepageUrl
       watchers {
         totalCount
       }
       stargazers {
         totalCount
       }
+      releases {
+        totalCount
+      }
+      deployments {
+        totalCount
+      }
+      environments {
+        totalCount
+      }
       languages(first: 8) {
         edges {
           size
@@ -24,6 +34,10 @@ query BaseRepository {
           }
         }
       }
+      licenseInfo {
+        name
+        spdxId
+      }
       issues_open: issues(states: OPEN) {
         totalCount
       }
@@ -39,20 +53,6 @@ query BaseRepository {
       pr_merged: pullRequests(states: MERGED) {
         totalCount
       }
-      releases {
-        totalCount
-      }
-      forkCount
-      licenseInfo {
-        name
-        spdxId
-      }
-      deployments {
-        totalCount
-      }
-      environments {
-        totalCount
-      }
     }
   }
 }

source/plugins/base/queries/user.graphql

@@ -9,56 +9,5 @@ query BaseUser {
     websiteUrl
     isHireable
     twitterUsername
-    repositories(last: 0 $forks $affiliations) {
-      totalCount
-      totalDiskUsage
-    }
-    packages {
-      totalCount
-    }
-    starredRepositories {
-      totalCount
-    }
-    watching {
-      totalCount
-    }
-    sponsorshipsAsSponsor {
-      totalCount
-    }
-    sponsorshipsAsMaintainer {
-      totalCount
-    }
-    contributionsCollection {
-      totalRepositoriesWithContributedCommits
-      totalCommitContributions
-      restrictedContributionsCount
-      totalIssueContributions
-      totalPullRequestContributions
-      totalPullRequestReviewContributions
-    }
-    calendar:contributionsCollection(from: "$calendar.from", to: "$calendar.to") {
-      contributionCalendar {
-        weeks {
-          contributionDays {
-            color
-          }
-        }
-      }
-    }
-    repositoriesContributedTo(includeUserRepositories: true) {
-      totalCount
-    }
-    followers {
-      totalCount
-    }
-    following {
-      totalCount
-    }
-    issueComments {
-      totalCount
-    }
-    organizations {
-      totalCount
-    }
   }
 }