sudacode/metrics
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(plugins/base): split base queries to reduce GitHub API timeouts (#584)

Browse Source
This commit is contained in:
Simon Lecoq
2021-10-22 00:05:00 -04:00
committed by GitHub
parent f75638d3a0
commit ead8bd2eb7
14 changed files with 223 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
source/app/mocks/api/github/graphql/base.calendar.mjs Normal file
View File

@@ -0,0 +1,41 @@
/**Mocked data */
export default function({faker, query, login = faker.internet.userName()}) {
console.debug("metrics/compute/mocks > mocking graphql api result > base/user")
return ({
user:{
calendar:{
contributionCalendar:{
weeks:[
{
contributionDays:[
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
],
},
{
contributionDays:[
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
],
},
{
contributionDays:[
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
],
},
],
},
},
},
})
}

16
source/app/mocks/api/github/graphql/base.contributions.mjs Normal file
View File

@@ -0,0 +1,16 @@
/**Mocked data */
export default function({faker, query, login = faker.internet.userName()}) {
console.debug("metrics/compute/mocks > mocking graphql api result > base/user")
return ({
user:{
contributionsCollection:{
totalRepositoriesWithContributedCommits:faker.datatype.number(100),
totalCommitContributions:faker.datatype.number(10000),
restrictedContributionsCount:faker.datatype.number(10000),
totalIssueContributions:faker.datatype.number(100),
totalPullRequestContributions:faker.datatype.number(1000),
totalPullRequestReviewContributions:faker.datatype.number(1000),
},
},
})
}

18
source/app/mocks/api/github/graphql/base.field.mjs Normal file
View File

@@ -0,0 +1,18 @@
/**Mocked data */
export default function({faker, query, login = faker.internet.userName()}) {
console.debug("metrics/compute/mocks > mocking graphql api result > base/user")
return ({
user:{
packages:{totalCount:faker.datatype.number(10)},
starredRepositories:{totalCount:faker.datatype.number(1000)},
watching:{totalCount:faker.datatype.number(100)},
sponsorshipsAsSponsor:{totalCount:faker.datatype.number(10)},
sponsorshipsAsMaintainer:{totalCount:faker.datatype.number(10)},
repositoriesContributedTo:{totalCount:faker.datatype.number(100)},
followers:{totalCount:faker.datatype.number(1000)},
following:{totalCount:faker.datatype.number(1000)},
issueComments:{totalCount:faker.datatype.number(1000)},
organizations:{totalCount:faker.datatype.number(10)},
},
})
}

9
source/app/mocks/api/github/graphql/base.field.repositories.mjs Normal file
View File

@@ -0,0 +1,9 @@
/**Mocked data */
export default function({faker, query, login = faker.internet.userName()}) {
console.debug("metrics/compute/mocks > mocking graphql api result > base/user")
return ({
user:{
repositories:{totalCount:faker.datatype.number(100), totalDiskUsage:faker.datatype.number(100000)},
},
})
}

52
source/app/mocks/api/github/graphql/base.user.mjs
View File

@@ -11,58 +11,6 @@ export default function({faker, query, login = faker.internet.userName()}) {
websiteUrl:faker.internet.url(),
isHireable:faker.datatype.boolean(),
twitterUsername:login,
repositories:{totalCount:faker.datatype.number(100), totalDiskUsage:faker.datatype.number(100000)},
packages:{totalCount:faker.datatype.number(10)},
starredRepositories:{totalCount:faker.datatype.number(1000)},
watching:{totalCount:faker.datatype.number(100)},
sponsorshipsAsSponsor:{totalCount:faker.datatype.number(10)},
sponsorshipsAsMaintainer:{totalCount:faker.datatype.number(10)},
contributionsCollection:{
totalRepositoriesWithContributedCommits:faker.datatype.number(100),
totalCommitContributions:faker.datatype.number(10000),
restrictedContributionsCount:faker.datatype.number(10000),
totalIssueContributions:faker.datatype.number(100),
totalPullRequestContributions:faker.datatype.number(1000),
totalPullRequestReviewContributions:faker.datatype.number(1000),
},
calendar:{
contributionCalendar:{
weeks:[
{
contributionDays:[
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
],
},
{
contributionDays:[
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
],
},
{
contributionDays:[
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
{color:faker.random.arrayElement(["#ebedf0", "#9be9a8", "#40c463", "#30a14e", "#216e39"])},
],
},
],
},
},
repositoriesContributedTo:{totalCount:faker.datatype.number(100)},
followers:{totalCount:faker.datatype.number(1000)},
following:{totalCount:faker.datatype.number(1000)},
issueComments:{totalCount:faker.datatype.number(1000)},
organizations:{totalCount:faker.datatype.number(10)},
},
})
}

92
source/plugins/base/index.mjs
View File

@@ -27,16 +27,65 @@ export default async function({login, graphql, rest, data, q, queries, imports},
try {
//Query data from GitHub API
console.debug(`metrics/compute/${login}/base > account ${account}`)
const queried = await graphql(queries.base[account]({login, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString(), forks, affiliations}))
const queried = await graphql(queries.base[account]({login}))
Object.assign(data, {user:queried[account]})
postprocess?.[account]({login, data})
//Query basic fields
const fields = {
user:["packages", "starredRepositories", "watching", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "followers", "following", "issueComments", "organizations", "repositoriesContributedTo(includeUserRepositories: true)"],
organization:["packages", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "membersWithRole"],
}[account] ?? []
for (const field of fields) {
try {
Object.assign(data.user, (await graphql(queries.base.field({login, account, field})))[account])
}
catch {
console.debug(`metrics/compute/${login}/base > failed to retrieve ${field}`)
data.user[field] = {totalCount:NaN}
}
}
//Query repositories fields
for (const field of ["totalCount", "totalDiskUsage"]) {
try {
Object.assign(data.user.repositories, (await graphql(queries.base["field.repositories"]({login, account, field})))[account].repositories)
}
catch (error) {
console.log(error)
console.debug(`metrics/compute/${login}/base > failed to retrieve repositories.${field}`)
data.user.repositories[field] = NaN
}
}
//Query user account fields
if (account === "user") {
//Query contributions collection
{
const fields = ["totalRepositoriesWithContributedCommits", "totalCommitContributions", "restrictedContributionsCount", "totalIssueContributions", "totalPullRequestContributions", "totalPullRequestReviewContributions"]
for (const field of fields) {
try {
Object.assign(data.user.contributionsCollection, (await graphql(queries.base.contributions({login, account, field})))[account].contributionsCollection)
}
catch {
console.debug(`metrics/compute/${login}/base > failed to retrieve contributionsCollection.${field}`)
data.user.contributionsCollection[field] = NaN
}
}
}
//Query calendar
try {
Object.assign(data.user, (await graphql(queries.base.calendar({login, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString()})))[account])
}
catch {
console.debug(`metrics/compute/${login}/base > failed to retrieve contributions calendar`)
data.user.calendar = {contributionCalendar:{weeks:[]}}
}
}
//Query repositories from GitHub API
data.user.repositoriesContributedTo.nodes = data.user.repositoriesContributedTo.nodes ?? []
for (const type of ({user:["repositories", "repositoriesContributedTo"], organization:["repositories"]}[account] ?? [])) {
//Iterate through repositories
let cursor = null
let pushed = 0
const options = {repositories:{forks, affiliations, constraints:""}, repositoriesContributedTo:{forks:"", affiliations:"", constraints:", includeUserRepositories: false, contributionTypes: COMMIT"}}[type] ?? null
data.user[type] = data.user[type] ?? {}
data.user[type].nodes = data.user[type].nodes ?? []
do {
console.debug(`metrics/compute/${login}/base > retrieving ${type} after ${cursor}`)
@@ -45,7 +94,7 @@ export default async function({login, graphql, rest, data, q, queries, imports},
data.user[type].nodes.push(...nodes)
pushed = nodes.length
console.debug(`metrics/compute/${login}/base > retrieved ${pushed} ${type} after ${cursor}`)
} while ((pushed) && (cursor) && (data.user.repositories.nodes.length + data.user.repositoriesContributedTo.nodes.length < repositories))
} while ((pushed) && (cursor) && ((data.user.repositories?.nodes?.length ?? 0) + (data.user.repositoriesContributedTo?.nodes?.length ?? 0) < repositories))
//Limit repositories
console.debug(`metrics/compute/${login}/base > keeping only ${repositories} ${type}`)
data.user[type].nodes.splice(repositories)
@@ -93,6 +142,8 @@ const postprocess = {
data.account = "user"
Object.assign(data.user, {
isVerified:false,
repositories:{},
contributionsCollection:{},
})
},
//Organization
@@ -101,22 +152,23 @@ const postprocess = {
data.account = "organization"
Object.assign(data.user, {
isHireable:false,
starredRepositories:{totalCount:0},
watching:{totalCount:0},
repositories:{},
starredRepositories:{totalCount:NaN},
watching:{totalCount:NaN},
contributionsCollection:{
totalRepositoriesWithContributedCommits:0,
totalCommitContributions:0,
restrictedContributionsCount:0,
totalIssueContributions:0,
totalPullRequestContributions:0,
totalPullRequestReviewContributions:0,
totalRepositoriesWithContributedCommits:NaN,
totalCommitContributions:NaN,
restrictedContributionsCount:NaN,
totalIssueContributions:NaN,
totalPullRequestContributions:NaN,
totalPullRequestReviewContributions:NaN,
},
calendar:{contributionCalendar:{weeks:[]}},
repositoriesContributedTo:{totalCount:0},
followers:{totalCount:0},
following:{totalCount:0},
issueComments:{totalCount:0},
organizations:{totalCount:0},
repositoriesContributedTo:{totalCount:NaN, nodes:[]},
followers:{totalCount:NaN},
following:{totalCount:NaN},
issueComments:{totalCount:NaN},
organizations:{totalCount:NaN},
})
},
//Skip base content query and instantiate an empty user instance
@@ -127,16 +179,16 @@ const postprocess = {
postprocess?.[account]({login, data})
data.account = "bypass"
Object.assign(data.user, {
databaseId:0,
databaseId:NaN,
name:login,
login,
createdAt:new Date(),
avatarUrl:`https://github.com/${login}.png`,
websiteUrl:null,
twitterUsername:login,
repositories:{totalCount:0, totalDiskUsage:0, nodes:[]},
packages:{totalCount:0},
repositoriesContributedTo:{nodes:[]},
repositories:{totalCount:NaN, totalDiskUsage:NaN, nodes:[]},
packages:{totalCount:NaN},
repositoriesContributedTo:{totalCount:NaN, nodes:[]},
})
},
}

13
source/plugins/base/queries/calendar.graphql Normal file
View File

@@ -0,0 +1,13 @@
query BaseCalendar {
user(login: "$login") {
calendar:contributionsCollection(from: "$calendar.from", to: "$calendar.to") {
contributionCalendar {
weeks {
contributionDays {
color
}
}
}
}
}
}

9
source/plugins/base/queries/contributions.graphql Normal file
View File

@@ -0,0 +1,9 @@
query BaseContributions {
user(login: "$login") {
contributionsCollection {
$field
}
}
}

7
source/plugins/base/queries/field.graphql Normal file
View File

@@ -0,0 +1,7 @@
query BaseField {
$account(login: "$login") {
$field {
totalCount
}
}
}

7
source/plugins/base/queries/field.repositories.graphql Normal file
View File

@@ -0,0 +1,7 @@
query BaseFieldRepositories {
$account(login: "$login") {
repositories(last: 0) {
$field
}
}
}

16
source/plugins/base/queries/organization.graphql
View File

@@ -9,21 +9,5 @@ query BaseOrganization {
websiteUrl
isVerified
twitterUsername
repositories(last: 0) {
totalCount
totalDiskUsage
}
packages {
totalCount
}
sponsorshipsAsSponsor {
totalCount
}
sponsorshipsAsMaintainer {
totalCount
}
membersWithRole {
totalCount
}
}
}

28
source/plugins/base/queries/repositories.graphql
View File

@@ -10,12 +10,22 @@ query BaseRepositories {
login
}
isFork
forkCount
watchers {
totalCount
}
stargazers {
totalCount
}
releases {
totalCount
}
deployments {
totalCount
}
environments {
totalCount
}
languages(first: 8) {
edges {
size
@@ -25,6 +35,10 @@ query BaseRepositories {
}
}
}
licenseInfo {
name
spdxId
}
issues_open: issues(states: OPEN) {
totalCount
}
@@ -40,20 +54,6 @@ query BaseRepositories {
pr_merged: pullRequests(states: MERGED) {
totalCount
}
releases {
totalCount
}
forkCount
licenseInfo {
name
spdxId
}
deployments {
totalCount
}
environments {
totalCount
}
}
}
}

34
source/plugins/base/queries/repository.graphql
View File

@@ -2,19 +2,29 @@ query BaseRepository {
$account(login: "$login") {
repository(name: "$repo") {
name
createdAt
diskUsage
homepageUrl
owner {
login
}
isFork
createdAt
diskUsage
homepageUrl
forkCount
watchers {
totalCount
}
stargazers {
totalCount
}
releases {
totalCount
}
deployments {
totalCount
}
environments {
totalCount
}
languages(first: 8) {
edges {
size
@@ -24,6 +34,10 @@ query BaseRepository {
}
}
}
licenseInfo {
name
spdxId
}
issues_open: issues(states: OPEN) {
totalCount
}
@@ -39,20 +53,6 @@ query BaseRepository {
pr_merged: pullRequests(states: MERGED) {
totalCount
}
releases {
totalCount
}
forkCount
licenseInfo {
name
spdxId
}
deployments {
totalCount
}
environments {
totalCount
}
}
}
}

51
source/plugins/base/queries/user.graphql
View File

@@ -9,56 +9,5 @@ query BaseUser {
websiteUrl
isHireable
twitterUsername
repositories(last: 0 $forks $affiliations) {
totalCount
totalDiskUsage
}
packages {
totalCount
}
starredRepositories {
totalCount
}
watching {
totalCount
}
sponsorshipsAsSponsor {
totalCount
}
sponsorshipsAsMaintainer {
totalCount
}
contributionsCollection {
totalRepositoriesWithContributedCommits
totalCommitContributions
restrictedContributionsCount
totalIssueContributions
totalPullRequestContributions
totalPullRequestReviewContributions
}
calendar:contributionsCollection(from: "$calendar.from", to: "$calendar.to") {
contributionCalendar {
weeks {
contributionDays {
color
}
}
}
}
repositoriesContributedTo(includeUserRepositories: true) {
totalCount
}
followers {
totalCount
}
following {
totalCount
}
issueComments {
totalCount
}
organizations {
totalCount
}
}
}