From ead8bd2eb797c0b3cdd1ecd41e561ecf781b4d4a Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Fri, 22 Oct 2021 00:05:00 -0400 Subject: [PATCH] feat(plugins/base): split base queries to reduce GitHub API timeouts (#584) --- .../api/github/graphql/base.calendar.mjs | 41 +++++++++ .../api/github/graphql/base.contributions.mjs | 16 ++++ .../mocks/api/github/graphql/base.field.mjs | 18 ++++ .../graphql/base.field.repositories.mjs | 9 ++ .../mocks/api/github/graphql/base.user.mjs | 52 ----------- source/plugins/base/index.mjs | 92 +++++++++++++++---- source/plugins/base/queries/calendar.graphql | 13 +++ .../base/queries/contributions.graphql | 9 ++ source/plugins/base/queries/field.graphql | 7 ++ .../base/queries/field.repositories.graphql | 7 ++ .../plugins/base/queries/organization.graphql | 16 ---- .../plugins/base/queries/repositories.graphql | 28 +++--- .../plugins/base/queries/repository.graphql | 34 +++---- source/plugins/base/queries/user.graphql | 51 ---------- 14 files changed, 223 insertions(+), 170 deletions(-) create mode 100644 source/app/mocks/api/github/graphql/base.calendar.mjs create mode 100644 source/app/mocks/api/github/graphql/base.contributions.mjs create mode 100644 source/app/mocks/api/github/graphql/base.field.mjs create mode 100644 source/app/mocks/api/github/graphql/base.field.repositories.mjs create mode 100644 source/plugins/base/queries/calendar.graphql create mode 100644 source/plugins/base/queries/contributions.graphql create mode 100644 source/plugins/base/queries/field.graphql create mode 100644 source/plugins/base/queries/field.repositories.graphql diff --git a/source/app/mocks/api/github/graphql/base.calendar.mjs b/source/app/mocks/api/github/graphql/base.calendar.mjs new file mode 100644 index 00000000..cd084d8d --- /dev/null +++ b/source/app/mocks/api/github/graphql/base.calendar.mjs @@ -0,0 +1,41 @@ +/**Mocked data */ +export default function({faker, query, login = faker.internet.userName()}) { + console.debug("metrics/compute/mocks > mocking graphql api result > base/user") + return ({ + user:{ + calendar:{ + contributionCalendar:{ + weeks:[ + { + contributionDays:[ + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + ], + }, + { + contributionDays:[ + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + ], + }, + { + contributionDays:[ + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, + ], + }, + ], + }, + }, + }, + }) +} diff --git a/source/app/mocks/api/github/graphql/base.contributions.mjs b/source/app/mocks/api/github/graphql/base.contributions.mjs new file mode 100644 index 00000000..d1b8db50 --- /dev/null +++ b/source/app/mocks/api/github/graphql/base.contributions.mjs @@ -0,0 +1,16 @@ +/**Mocked data */ +export default function({faker, query, login = faker.internet.userName()}) { + console.debug("metrics/compute/mocks > mocking graphql api result > base/user") + return ({ + user:{ + contributionsCollection:{ + totalRepositoriesWithContributedCommits:faker.datatype.number(100), + totalCommitContributions:faker.datatype.number(10000), + restrictedContributionsCount:faker.datatype.number(10000), + totalIssueContributions:faker.datatype.number(100), + totalPullRequestContributions:faker.datatype.number(1000), + totalPullRequestReviewContributions:faker.datatype.number(1000), + }, + }, + }) +} diff --git a/source/app/mocks/api/github/graphql/base.field.mjs b/source/app/mocks/api/github/graphql/base.field.mjs new file mode 100644 index 00000000..84321107 --- /dev/null +++ b/source/app/mocks/api/github/graphql/base.field.mjs @@ -0,0 +1,18 @@ +/**Mocked data */ +export default function({faker, query, login = faker.internet.userName()}) { + console.debug("metrics/compute/mocks > mocking graphql api result > base/user") + return ({ + user:{ + packages:{totalCount:faker.datatype.number(10)}, + starredRepositories:{totalCount:faker.datatype.number(1000)}, + watching:{totalCount:faker.datatype.number(100)}, + sponsorshipsAsSponsor:{totalCount:faker.datatype.number(10)}, + sponsorshipsAsMaintainer:{totalCount:faker.datatype.number(10)}, + repositoriesContributedTo:{totalCount:faker.datatype.number(100)}, + followers:{totalCount:faker.datatype.number(1000)}, + following:{totalCount:faker.datatype.number(1000)}, + issueComments:{totalCount:faker.datatype.number(1000)}, + organizations:{totalCount:faker.datatype.number(10)}, + }, + }) +} diff --git a/source/app/mocks/api/github/graphql/base.field.repositories.mjs b/source/app/mocks/api/github/graphql/base.field.repositories.mjs new file mode 100644 index 00000000..35a049e3 --- /dev/null +++ b/source/app/mocks/api/github/graphql/base.field.repositories.mjs @@ -0,0 +1,9 @@ +/**Mocked data */ +export default function({faker, query, login = faker.internet.userName()}) { + console.debug("metrics/compute/mocks > mocking graphql api result > base/user") + return ({ + user:{ + repositories:{totalCount:faker.datatype.number(100), totalDiskUsage:faker.datatype.number(100000)}, + }, + }) +} diff --git a/source/app/mocks/api/github/graphql/base.user.mjs b/source/app/mocks/api/github/graphql/base.user.mjs index 19edc87e..28c4d991 100644 --- a/source/app/mocks/api/github/graphql/base.user.mjs +++ b/source/app/mocks/api/github/graphql/base.user.mjs @@ -11,58 +11,6 @@ export default function({faker, query, login = faker.internet.userName()}) { websiteUrl:faker.internet.url(), isHireable:faker.datatype.boolean(), twitterUsername:login, - repositories:{totalCount:faker.datatype.number(100), totalDiskUsage:faker.datatype.number(100000)}, - packages:{totalCount:faker.datatype.number(10)}, - starredRepositories:{totalCount:faker.datatype.number(1000)}, - watching:{totalCount:faker.datatype.number(100)}, - sponsorshipsAsSponsor:{totalCount:faker.datatype.number(10)}, - sponsorshipsAsMaintainer:{totalCount:faker.datatype.number(10)}, - contributionsCollection:{ - totalRepositoriesWithContributedCommits:faker.datatype.number(100), - totalCommitContributions:faker.datatype.number(10000), - restrictedContributionsCount:faker.datatype.number(10000), - totalIssueContributions:faker.datatype.number(100), - totalPullRequestContributions:faker.datatype.number(1000), - totalPullRequestReviewContributions:faker.datatype.number(1000), - }, - calendar:{ - contributionCalendar:{ - weeks:[ - { - contributionDays:[ - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - ], - }, - { - contributionDays:[ - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - ], - }, - { - contributionDays:[ - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - {color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])}, - ], - }, - ], - }, - }, - repositoriesContributedTo:{totalCount:faker.datatype.number(100)}, - followers:{totalCount:faker.datatype.number(1000)}, - following:{totalCount:faker.datatype.number(1000)}, - issueComments:{totalCount:faker.datatype.number(1000)}, - organizations:{totalCount:faker.datatype.number(10)}, }, }) } diff --git a/source/plugins/base/index.mjs b/source/plugins/base/index.mjs index 9d2a50db..8aaffd48 100644 --- a/source/plugins/base/index.mjs +++ b/source/plugins/base/index.mjs @@ -27,16 +27,65 @@ export default async function({login, graphql, rest, data, q, queries, imports}, try { //Query data from GitHub API console.debug(`metrics/compute/${login}/base > account ${account}`) - const queried = await graphql(queries.base[account]({login, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString(), forks, affiliations})) + const queried = await graphql(queries.base[account]({login})) Object.assign(data, {user:queried[account]}) postprocess?.[account]({login, data}) + //Query basic fields + const fields = { + user:["packages", "starredRepositories", "watching", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "followers", "following", "issueComments", "organizations", "repositoriesContributedTo(includeUserRepositories: true)"], + organization:["packages", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "membersWithRole"], + }[account] ?? [] + for (const field of fields) { + try { + Object.assign(data.user, (await graphql(queries.base.field({login, account, field})))[account]) + } + catch { + console.debug(`metrics/compute/${login}/base > failed to retrieve ${field}`) + data.user[field] = {totalCount:NaN} + } + } + //Query repositories fields + for (const field of ["totalCount", "totalDiskUsage"]) { + try { + Object.assign(data.user.repositories, (await graphql(queries.base["field.repositories"]({login, account, field})))[account].repositories) + } + catch (error) { + console.log(error) + console.debug(`metrics/compute/${login}/base > failed to retrieve repositories.${field}`) + data.user.repositories[field] = NaN + } + } + //Query user account fields + if (account === "user") { + //Query contributions collection + { + const fields = ["totalRepositoriesWithContributedCommits", "totalCommitContributions", "restrictedContributionsCount", "totalIssueContributions", "totalPullRequestContributions", "totalPullRequestReviewContributions"] + for (const field of fields) { + try { + Object.assign(data.user.contributionsCollection, (await graphql(queries.base.contributions({login, account, field})))[account].contributionsCollection) + } + catch { + console.debug(`metrics/compute/${login}/base > failed to retrieve contributionsCollection.${field}`) + data.user.contributionsCollection[field] = NaN + } + } + } + //Query calendar + try { + Object.assign(data.user, (await graphql(queries.base.calendar({login, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString()})))[account]) + } + catch { + console.debug(`metrics/compute/${login}/base > failed to retrieve contributions calendar`) + data.user.calendar = {contributionCalendar:{weeks:[]}} + } + } //Query repositories from GitHub API - data.user.repositoriesContributedTo.nodes = data.user.repositoriesContributedTo.nodes ?? [] for (const type of ({user:["repositories", "repositoriesContributedTo"], organization:["repositories"]}[account] ?? [])) { //Iterate through repositories let cursor = null let pushed = 0 const options = {repositories:{forks, affiliations, constraints:""}, repositoriesContributedTo:{forks:"", affiliations:"", constraints:", includeUserRepositories: false, contributionTypes: COMMIT"}}[type] ?? null + data.user[type] = data.user[type] ?? {} data.user[type].nodes = data.user[type].nodes ?? [] do { console.debug(`metrics/compute/${login}/base > retrieving ${type} after ${cursor}`) @@ -45,7 +94,7 @@ export default async function({login, graphql, rest, data, q, queries, imports}, data.user[type].nodes.push(...nodes) pushed = nodes.length console.debug(`metrics/compute/${login}/base > retrieved ${pushed} ${type} after ${cursor}`) - } while ((pushed) && (cursor) && (data.user.repositories.nodes.length + data.user.repositoriesContributedTo.nodes.length < repositories)) + } while ((pushed) && (cursor) && ((data.user.repositories?.nodes?.length ?? 0) + (data.user.repositoriesContributedTo?.nodes?.length ?? 0) < repositories)) //Limit repositories console.debug(`metrics/compute/${login}/base > keeping only ${repositories} ${type}`) data.user[type].nodes.splice(repositories) @@ -93,6 +142,8 @@ const postprocess = { data.account = "user" Object.assign(data.user, { isVerified:false, + repositories:{}, + contributionsCollection:{}, }) }, //Organization @@ -101,22 +152,23 @@ const postprocess = { data.account = "organization" Object.assign(data.user, { isHireable:false, - starredRepositories:{totalCount:0}, - watching:{totalCount:0}, + repositories:{}, + starredRepositories:{totalCount:NaN}, + watching:{totalCount:NaN}, contributionsCollection:{ - totalRepositoriesWithContributedCommits:0, - totalCommitContributions:0, - restrictedContributionsCount:0, - totalIssueContributions:0, - totalPullRequestContributions:0, - totalPullRequestReviewContributions:0, + totalRepositoriesWithContributedCommits:NaN, + totalCommitContributions:NaN, + restrictedContributionsCount:NaN, + totalIssueContributions:NaN, + totalPullRequestContributions:NaN, + totalPullRequestReviewContributions:NaN, }, calendar:{contributionCalendar:{weeks:[]}}, - repositoriesContributedTo:{totalCount:0}, - followers:{totalCount:0}, - following:{totalCount:0}, - issueComments:{totalCount:0}, - organizations:{totalCount:0}, + repositoriesContributedTo:{totalCount:NaN, nodes:[]}, + followers:{totalCount:NaN}, + following:{totalCount:NaN}, + issueComments:{totalCount:NaN}, + organizations:{totalCount:NaN}, }) }, //Skip base content query and instantiate an empty user instance @@ -127,16 +179,16 @@ const postprocess = { postprocess?.[account]({login, data}) data.account = "bypass" Object.assign(data.user, { - databaseId:0, + databaseId:NaN, name:login, login, createdAt:new Date(), avatarUrl:`https://github.com/${login}.png`, websiteUrl:null, twitterUsername:login, - repositories:{totalCount:0, totalDiskUsage:0, nodes:[]}, - packages:{totalCount:0}, - repositoriesContributedTo:{nodes:[]}, + repositories:{totalCount:NaN, totalDiskUsage:NaN, nodes:[]}, + packages:{totalCount:NaN}, + repositoriesContributedTo:{totalCount:NaN, nodes:[]}, }) }, } diff --git a/source/plugins/base/queries/calendar.graphql b/source/plugins/base/queries/calendar.graphql new file mode 100644 index 00000000..df4a79e4 --- /dev/null +++ b/source/plugins/base/queries/calendar.graphql @@ -0,0 +1,13 @@ +query BaseCalendar { + user(login: "$login") { + calendar:contributionsCollection(from: "$calendar.from", to: "$calendar.to") { + contributionCalendar { + weeks { + contributionDays { + color + } + } + } + } + } +} \ No newline at end of file diff --git a/source/plugins/base/queries/contributions.graphql b/source/plugins/base/queries/contributions.graphql new file mode 100644 index 00000000..54a3fd4a --- /dev/null +++ b/source/plugins/base/queries/contributions.graphql @@ -0,0 +1,9 @@ +query BaseContributions { + user(login: "$login") { + contributionsCollection { + $field + } + } +} + + diff --git a/source/plugins/base/queries/field.graphql b/source/plugins/base/queries/field.graphql new file mode 100644 index 00000000..2939326c --- /dev/null +++ b/source/plugins/base/queries/field.graphql @@ -0,0 +1,7 @@ +query BaseField { + $account(login: "$login") { + $field { + totalCount + } + } +} \ No newline at end of file diff --git a/source/plugins/base/queries/field.repositories.graphql b/source/plugins/base/queries/field.repositories.graphql new file mode 100644 index 00000000..faff5faf --- /dev/null +++ b/source/plugins/base/queries/field.repositories.graphql @@ -0,0 +1,7 @@ +query BaseFieldRepositories { + $account(login: "$login") { + repositories(last: 0) { + $field + } + } +} \ No newline at end of file diff --git a/source/plugins/base/queries/organization.graphql b/source/plugins/base/queries/organization.graphql index 3339e897..fb7a68af 100644 --- a/source/plugins/base/queries/organization.graphql +++ b/source/plugins/base/queries/organization.graphql @@ -9,21 +9,5 @@ query BaseOrganization { websiteUrl isVerified twitterUsername - repositories(last: 0) { - totalCount - totalDiskUsage - } - packages { - totalCount - } - sponsorshipsAsSponsor { - totalCount - } - sponsorshipsAsMaintainer { - totalCount - } - membersWithRole { - totalCount - } } } diff --git a/source/plugins/base/queries/repositories.graphql b/source/plugins/base/queries/repositories.graphql index 443260f0..d5b25973 100644 --- a/source/plugins/base/queries/repositories.graphql +++ b/source/plugins/base/queries/repositories.graphql @@ -10,12 +10,22 @@ query BaseRepositories { login } isFork + forkCount watchers { totalCount } stargazers { totalCount } + releases { + totalCount + } + deployments { + totalCount + } + environments { + totalCount + } languages(first: 8) { edges { size @@ -25,6 +35,10 @@ query BaseRepositories { } } } + licenseInfo { + name + spdxId + } issues_open: issues(states: OPEN) { totalCount } @@ -40,20 +54,6 @@ query BaseRepositories { pr_merged: pullRequests(states: MERGED) { totalCount } - releases { - totalCount - } - forkCount - licenseInfo { - name - spdxId - } - deployments { - totalCount - } - environments { - totalCount - } } } } diff --git a/source/plugins/base/queries/repository.graphql b/source/plugins/base/queries/repository.graphql index 9cbf0bd3..a9094af9 100644 --- a/source/plugins/base/queries/repository.graphql +++ b/source/plugins/base/queries/repository.graphql @@ -2,19 +2,29 @@ query BaseRepository { $account(login: "$login") { repository(name: "$repo") { name + createdAt + diskUsage + homepageUrl owner { login } isFork - createdAt - diskUsage - homepageUrl + forkCount watchers { totalCount } stargazers { totalCount } + releases { + totalCount + } + deployments { + totalCount + } + environments { + totalCount + } languages(first: 8) { edges { size @@ -24,6 +34,10 @@ query BaseRepository { } } } + licenseInfo { + name + spdxId + } issues_open: issues(states: OPEN) { totalCount } @@ -39,20 +53,6 @@ query BaseRepository { pr_merged: pullRequests(states: MERGED) { totalCount } - releases { - totalCount - } - forkCount - licenseInfo { - name - spdxId - } - deployments { - totalCount - } - environments { - totalCount - } } } } \ No newline at end of file diff --git a/source/plugins/base/queries/user.graphql b/source/plugins/base/queries/user.graphql index ada9b190..89284de5 100644 --- a/source/plugins/base/queries/user.graphql +++ b/source/plugins/base/queries/user.graphql @@ -9,56 +9,5 @@ query BaseUser { websiteUrl isHireable twitterUsername - repositories(last: 0 $forks $affiliations) { - totalCount - totalDiskUsage - } - packages { - totalCount - } - starredRepositories { - totalCount - } - watching { - totalCount - } - sponsorshipsAsSponsor { - totalCount - } - sponsorshipsAsMaintainer { - totalCount - } - contributionsCollection { - totalRepositoriesWithContributedCommits - totalCommitContributions - restrictedContributionsCount - totalIssueContributions - totalPullRequestContributions - totalPullRequestReviewContributions - } - calendar:contributionsCollection(from: "$calendar.from", to: "$calendar.to") { - contributionCalendar { - weeks { - contributionDays { - color - } - } - } - } - repositoriesContributedTo(includeUserRepositories: true) { - totalCount - } - followers { - totalCount - } - following { - totalCount - } - issueComments { - totalCount - } - organizations { - totalCount - } } }