diff --git a/source/app/metrics/metadata.mjs b/source/app/metrics/metadata.mjs index a4ae5e87..41f4b35a 100644 --- a/source/app/metrics/metadata.mjs +++ b/source/app/metrics/metadata.mjs @@ -84,11 +84,11 @@ metadata.plugin = async function({__plugins, name, logger}) { value = `${value}`.trim() if (user) { if (value === ".user.login") - return user.login + return user.login ?? value if (value === ".user.twitter") - return user.twitterUsername + return user.twitterUsername ?? value if (value === ".user.website") - return user.websiteUrl + return user.websiteUrl ?? value } return value } @@ -133,7 +133,7 @@ metadata.plugin = async function({__plugins, name, logger}) { } const separators = {"comma-separated":",", "space-separated":" "} const separator = separators[[format].flat().filter(s => s in separators)[0]] ?? "," - return value.split(separator).map(v => replacer(v.trim()).toLocaleLowerCase()).filter(v => Array.isArray(values) ? values.includes(v) : true).filter(v => v) + return value.split(separator).map(v => replacer(v).toLocaleLowerCase()).filter(v => Array.isArray(values) ? values.includes(v) : true).filter(v => v) } //String case "string": { diff --git a/source/plugins/base/index.mjs b/source/plugins/base/index.mjs index be6ba675..b0196af5 100644 --- a/source/plugins/base/index.mjs +++ b/source/plugins/base/index.mjs @@ -7,14 +7,14 @@ export default async function({login, graphql, data, q, queries, imports}, conf) { //Load inputs console.debug(`metrics/compute/${login}/base > started`) - let {repositories, "repositories.forks":_forks, "repositories.affiliations":_affiliations} = imports.metadata.plugins.base.inputs({data, q, account:"bypass"}, {repositories:conf.settings.repositories ?? 100}) + let {repositories, "repositories.forks":_forks, "repositories.affiliations":_affiliations, "repositories.batch":_batch} = imports.metadata.plugins.base.inputs({data, q, account:"bypass"}, {repositories:conf.settings.repositories ?? 100}) const forks = _forks ? "" : ", isFork: false" const affiliations = _affiliations?.length ? `, ownerAffiliations: [${_affiliations.map(x => x.toLocaleUpperCase()).join(", ")}]${conf.authenticated === login ? `, affiliations: [${_affiliations.map(x => x.toLocaleUpperCase()).join(", ")}]` : ""}` : "" console.debug(`metrics/compute/${login}/base > affiliations constraints ${affiliations}`) //Skip initial data gathering if not needed if (conf.settings.notoken) - return (postprocess.skip({login, data}), {}) + return (postprocess.skip({login, data, imports}), {}) //Base parts (legacy handling for web instance) const defaulted = ("base" in q) ? legacy.converter(q.base) ?? true : true @@ -38,7 +38,7 @@ export default async function({login, graphql, data, q, queries, imports}, conf) const options = {repositories:{forks, affiliations, constraints:""}, repositoriesContributedTo:{forks:"", affiliations:"", constraints:", includeUserRepositories: false, contributionTypes: COMMIT"}}[type] ?? null do { console.debug(`metrics/compute/${login}/base > retrieving ${type} after ${cursor}`) - const {[account]:{[type]:{edges = [], nodes = []} = {}}} = await graphql(queries.base.repositories({login, account, type, after:cursor ? `after: "${cursor}"` : "", repositories:Math.min(repositories, {user:100, organization:25}[account]), ...options})) + const {[account]:{[type]:{edges = [], nodes = []} = {}}} = await graphql(queries.base.repositories({login, account, type, after:cursor ? `after: "${cursor}"` : "", repositories:Math.min(repositories, {user:_batch, organization:Math.min(25, _batch)}[account]), ...options})) cursor = edges?.[edges?.length - 1]?.cursor data.user[type].nodes.push(...nodes) pushed = nodes.length @@ -51,7 +51,7 @@ export default async function({login, graphql, data, q, queries, imports}, conf) } //Shared options let {"repositories.skipped":skipped, "commits.authoring":authoring} = imports.metadata.plugins.base.inputs({data, q, account:"bypass"}) - data.shared = {"repositories.skipped":skipped, "commits.authoring":authoring} + data.shared = {"repositories.skipped":skipped, "commits.authoring":authoring, "repositories.batch":_batch} console.debug(`metrics/compute/${login}/base > shared options > ${JSON.stringify(data.shared)}`) //Success console.debug(`metrics/compute/${login}/base > graphql query > account ${account} > success`) @@ -107,8 +107,9 @@ const postprocess = { }) }, //Skip base content query and instantiate an empty user instance - skip({login, data}) { + skip({login, data, imports}) { data.user = {} + data.shared = imports.metadata.plugins.base.inputs({data, q:{}, account:"bypass"}) for (const account of ["user", "organization"]) postprocess?.[account]({login, data}) data.account = "bypass" @@ -122,6 +123,7 @@ const postprocess = { twitterUsername:login, repositories:{totalCount:0, totalDiskUsage:0, nodes:[]}, packages:{totalCount:0}, + repositoriesContributedTo:{nodes:[]}, }) }, } diff --git a/source/plugins/base/metadata.yml b/source/plugins/base/metadata.yml index 3ac1b414..a4f3069f 100644 --- a/source/plugins/base/metadata.yml +++ b/source/plugins/base/metadata.yml @@ -28,6 +28,16 @@ inputs: default: 100 min: 0 + + # Number of repositories to load at once by queries + # If you encounter GitHub queries timeouts, using a lower value here may solve issues + repositories_batch: + description: Number of repositories to load at once by queries + type: number + default: 100 + max: 100 + min: 1 + # Include forked repositories into metrics repositories_forks: description: Include forks in metrics diff --git a/source/plugins/notable/index.mjs b/source/plugins/notable/index.mjs index 408cde1c..98a84861 100644 --- a/source/plugins/notable/index.mjs +++ b/source/plugins/notable/index.mjs @@ -18,7 +18,7 @@ export default async function({login, q, imports, graphql, data, account, querie let pushed = 0 do { console.debug(`metrics/compute/${login}/plugins > notable > retrieving contributed repositories after ${cursor}`) - const {user:{repositoriesContributedTo:{edges}}} = await graphql(queries.notable.contributions({login, after:cursor ? `after: "${cursor}"` : "", repositories:100})) + const {user:{repositoriesContributedTo:{edges}}} = await graphql(queries.notable.contributions({login, after:cursor ? `after: "${cursor}"` : "", repositories:data.shared["repositories.batch"] || 100})) cursor = edges?.[edges?.length - 1]?.cursor edges .filter(({node}) => node.isInOrganization)