diff --git a/source/app/action/index.mjs b/source/app/action/index.mjs index 0da5ef5a..b8460ad3 100644 --- a/source/app/action/index.mjs +++ b/source/app/action/index.mjs @@ -42,6 +42,28 @@ async function wait(seconds) { await new Promise(solve => setTimeout(solve, seconds * 1000)) } +//Retry wrapper +async function retry(func, {retries = 1, delay = 0} = {}) { + let error = null + for (let attempt = 1; attempt <= retries; attempt++) { + try { + console.debug(`::group::Attempt ${attempt}/${retries}`) + const result = await func() + console.debug("::endgroup::") + return result + } + catch (_error) { + error = _error + console.debug("::endgroup::") + console.debug(`::warning::${error.message}`) + await wait(delay) + } + } + if (error) + throw error + return null +} + //Runner (async function() { try { @@ -92,6 +114,8 @@ async function wait(seconds) { "use.prebuilt.image":_image, retries, "retries.delay":retries_delay, + "retries.output.action":retries_output_action, + "retries.delay.output.action":retries_delay_output_action, "output.action":_action, "output.condition":_output_condition, delay, @@ -303,23 +327,12 @@ async function wait(seconds) { //Render metrics info.break() info.section("Rendering") - let error = null, rendered = null - for (let attempt = 1; attempt <= retries; attempt++) { - try { - console.debug(`::group::Attempt ${attempt}/${retries}`) - ;({rendered} = await metrics({login:user, q}, {graphql, rest, plugins, conf, die, verify, convert}, {Plugins, Templates})) - console.debug("::endgroup::") - break - } - catch (_error) { - error = _error - console.debug("::endgroup::") - console.debug(`::warning::rendering failed (${error.message})`) - await wait(retries_delay) - } - } + let rendered = await retry(async () => { + const {rendered} = await metrics({login:user, q}, {graphql, rest, plugins, conf, die, verify, convert}, {Plugins, Templates}) + return rendered + }, {retries, delay:retries_delay}) if (!rendered) - throw error ?? new Error("Could not render metrics") + throw new Error("Could not render metrics") info("Status", "complete") //Output condition @@ -329,13 +342,15 @@ async function wait(seconds) { if ((_output_condition === "data-changed")&&((committer.commit) || (committer.pr))) { const {svg} = await import("../metrics/utils.mjs") let data = "" - try { - data = `${Buffer.from((await committer.rest.repos.getContent({...github.context.repo, ref:`heads/${committer.head}`, path:filename})).data.content, "base64")}` - } - catch (error) { - if (error.response.status !== 404) - throw error - } + await retry(async () => { + try { + data = `${Buffer.from((await committer.rest.repos.getContent({...github.context.repo, ref:`heads/${committer.head}`, path:filename})).data.content, "base64")}` + } + catch (error) { + if (error.response.status !== 404) + throw error + } + }, {retries:retries_output_action, delay:retries_delay_output_action}) const previous = await svg.hash(data) info("Previous hash", previous) const current = await svg.hash(rendered) @@ -363,43 +378,45 @@ async function wait(seconds) { process.exit(0) } - //Cache + //Cache embed svg for markdown outputs if (/markdown/.test(convert)) { const regex = /(?)/ let matched = null while (matched = regex.exec(rendered)?.groups) { //eslint-disable-line no-cond-assign - const {match, name, format, content} = matched - let path = `${_markdown_cache}/${name}.${format.replace(/[+].*$/g, "")}` - console.debug(`Processing ${path}`) - let sha = null - try { - const {repository:{object:{oid}}} = await graphql( - ` - query Sha { - repository(owner: "${github.context.repo.owner}", name: "${github.context.repo.repo}") { - object(expression: "${committer.head}:${path}") { ... on Blob { oid } } + await retry(async () => { + const {match, name, format, content} = matched + let path = `${_markdown_cache}/${name}.${format.replace(/[+].*$/g, "")}` + console.debug(`Processing ${path}`) + let sha = null + try { + const {repository:{object:{oid}}} = await graphql( + ` + query Sha { + repository(owner: "${github.context.repo.owner}", name: "${github.context.repo.repo}") { + object(expression: "${committer.head}:${path}") { ... on Blob { oid } } + } } - } - `, - {headers:{authorization:`token ${committer.token}`}}, - ) - sha = oid - } - catch (error) { - console.debug(error) - } - finally { - await committer.rest.repos.createOrUpdateFileContents({ - ...github.context.repo, - path, - content, - message:`${committer.message} (cache)`, - ...(sha ? {sha} : {}), - branch:committer.pr ? committer.head : committer.branch, - }) - rendered = rendered.replace(match, ``) - info(`Saving ${path}`, "ok") - } + `, + {headers:{authorization:`token ${committer.token}`}}, + ) + sha = oid + } + catch (error) { + console.debug(error) + } + finally { + await committer.rest.repos.createOrUpdateFileContents({ + ...github.context.repo, + path, + content, + message:`${committer.message} (cache)`, + ...(sha ? {sha} : {}), + branch:committer.pr ? committer.head : committer.branch, + }) + rendered = rendered.replace(match, ``) + info(`Saving ${path}`, "ok") + } + }, {retries:retries_output_action, delay:retries_delay_output_action}) } } @@ -416,84 +433,96 @@ async function wait(seconds) { //Upload to gist (this is done as user since committer_token may not have gist rights) if (committer.gist) { - await rest.gists.update({gist_id:committer.gist, files:{[filename]:{content:rendered}}}) - info(`Upload to gist ${committer.gist}`, "ok") - committer.commit = false + await retry(async () => { + await rest.gists.update({gist_id:committer.gist, files:{[filename]:{content:rendered}}}) + info(`Upload to gist ${committer.gist}`, "ok") + committer.commit = false + }, {retries:retries_output_action, delay:retries_delay_output_action}) } //Commit metrics if (committer.commit) { - await committer.rest.repos.createOrUpdateFileContents({ - ...github.context.repo, - path:filename, - message:committer.message, - content:Buffer.from(typeof rendered === "object" ? JSON.stringify(rendered) : `${rendered}`).toString("base64"), - branch:committer.pr ? committer.head : committer.branch, - ...(committer.sha ? {sha:committer.sha} : {}), - }) - info(`Commit to branch ${committer.branch}`, "ok") + await retry(async () => { + await committer.rest.repos.createOrUpdateFileContents({ + ...github.context.repo, + path:filename, + message:committer.message, + content:Buffer.from(typeof rendered === "object" ? JSON.stringify(rendered) : `${rendered}`).toString("base64"), + branch:committer.pr ? committer.head : committer.branch, + ...(committer.sha ? {sha:committer.sha} : {}), + }) + info(`Commit to branch ${committer.branch}`, "ok") + }, {retries:retries_output_action, delay:retries_delay_output_action}) } //Pull request if (committer.pr) { //Create pull request let number = null - try { - ({data:{number}} = await committer.rest.pulls.create({...github.context.repo, head:committer.head, base:committer.branch, title:`Auto-generated metrics for run #${github.context.runId}`, body:" ", maintainer_can_modify:true})) - info(`Pull request from ${committer.head} to ${committer.branch}`, "(created)") - } - catch (error) { - console.debug(error) - //Check if pull request has already been created previously - if (/A pull request already exists/.test(error)) { - info(`Pull request from ${committer.head} to ${committer.branch}`, "(already existing)") - const q = `repo:${github.context.repo.owner}/${github.context.repo.repo}+type:pr+state:open+Auto-generated metrics for run #${github.context.runId}+in:title` - const prs = (await committer.rest.search.issuesAndPullRequests({q})).data.items.filter(({user:{login}}) => login === "github-actions[bot]") - if (prs.length < 1) - throw new Error("0 matching prs. Cannot proceed.") - if (prs.length > 1) - throw new Error(`Found more than one matching prs: ${prs.map(({number}) => `#${number}`).join(", ")}. Cannot proceed.`) - ;({number} = prs.shift()) + await retry(async () => { + try { + ({data:{number}} = await committer.rest.pulls.create({...github.context.repo, head:committer.head, base:committer.branch, title:`Auto-generated metrics for run #${github.context.runId}`, body:" ", maintainer_can_modify:true})) + info(`Pull request from ${committer.head} to ${committer.branch}`, "(created)") } - //Check if pull request could not been created because there are no diff between head and base - else if (/No commits between/.test(error)) { - info(`Pull request from ${committer.head} to ${committer.branch}`, "(no diff)") - committer.merge = false - number = "(none)" + catch (error) { + console.debug(error) + //Check if pull request has already been created previously + if (/A pull request already exists/.test(error)) { + info(`Pull request from ${committer.head} to ${committer.branch}`, "(already existing)") + const q = `repo:${github.context.repo.owner}/${github.context.repo.repo}+type:pr+state:open+Auto-generated metrics for run #${github.context.runId}+in:title` + const prs = (await committer.rest.search.issuesAndPullRequests({q})).data.items.filter(({user:{login}}) => login === "github-actions[bot]") + if (prs.length < 1) + throw new Error("0 matching prs. Cannot proceed.") + if (prs.length > 1) + throw new Error(`Found more than one matching prs: ${prs.map(({number}) => `#${number}`).join(", ")}. Cannot proceed.`) + ;({number} = prs.shift()) + } + //Check if pull request could not been created because there are no diff between head and base + else if (/No commits between/.test(error)) { + info(`Pull request from ${committer.head} to ${committer.branch}`, "(no diff)") + committer.merge = false + number = "(none)" + } + else + throw error } - else - throw error - - } - info("Pull request number", number) + info("Pull request number", number) + }, {retries:retries_output_action, delay:retries_delay_output_action}) //Merge pull request if (committer.merge) { info("Merge method", committer.merge) let attempts = 240 do { - //Check pull request mergeability (https://octokit.github.io/rest.js/v18#pulls-get) - const {data:{mergeable, mergeable_state:state}} = await committer.rest.pulls.get({...github.context.repo, pull_number:number}) - console.debug(`Pull request #${number} mergeable state is "${state}"`) - if (mergeable === null) { - await wait(15) + const success = await retry(async () => { + //Check pull request mergeability (https://octokit.github.io/rest.js/v18#pulls-get) + const {data:{mergeable, mergeable_state:state}} = await committer.rest.pulls.get({...github.context.repo, pull_number:number}) + console.debug(`Pull request #${number} mergeable state is "${state}"`) + if (mergeable === null) { + await wait(15) + return false + } + if (!mergeable) + throw new Error(`Pull request #${number} is not mergeable (state is "${state}")`) + //Merge pull request + await committer.rest.pulls.merge({...github.context.repo, pull_number:number, merge_method:committer.merge}) + info(`Merge #${number} to ${committer.branch}`, "ok") + return true + }, {retries:retries_output_action, delay:retries_delay_output_action}) + if (!success) continue - } - if (!mergeable) - throw new Error(`Pull request #${number} is not mergeable (state is "${state}")`) - //Merge pull request - await committer.rest.pulls.merge({...github.context.repo, pull_number:number, merge_method:committer.merge}) - info(`Merge #${number} to ${committer.branch}`, "ok") //Delete head branch - try { - await wait(15) - await committer.rest.git.deleteRef({...github.context.repo, ref:`heads/${committer.head}`}) - } - catch (error) { - console.debug(error) - if (!/reference does not exist/i.test(`${error}`)) - throw error - } - info(`Branch ${committer.head}`, "(deleted)") + await retry(async () => { + try { + await wait(15) + await committer.rest.git.deleteRef({...github.context.repo, ref:`heads/${committer.head}`}) + } + catch (error) { + console.debug(error) + if (!/reference does not exist/i.test(`${error}`)) + throw error + } + info(`Branch ${committer.head}`, "(deleted)") + }, {retries:retries_output_action, delay:retries_delay_output_action}) break } while (--attempts) } diff --git a/source/plugins/core/README.md b/source/plugins/core/README.md index 43fda87c..e86aa995 100644 --- a/source/plugins/core/README.md +++ b/source/plugins/core/README.md @@ -228,11 +228,14 @@ It also possible to alter output condition using `output_condition` option, whic output_action: pull-request-merge ``` -### ♻️ Retrying automatically failed rendering +### ♻️ Retrying automatically failed rendering and output action Rendering is subject to external factors and can fail from time to time. It is possible to mitigate this issue using `retries` and `retries_delay` options to automatically retry later metrics rendering and avoid workflow fails. +Output action is also subject to GitHub API rate-limiting and status and can fail from time to time. +It is possible to mitigate this issue using `retries_output_action` and `retries_delay_output_action` options to automatically retry later metrics output action and avoid workflow fails. As this is a separate step from rendering, metrics rendering won't be computed again during this phase. + #### ℹ️ Examples workflows ```yaml @@ -241,6 +244,8 @@ It is possible to mitigate this issue using `retries` and `retries_delay` option # ... other options retries: 3 retries_delay: 300 + retries_output_action: 5 + retries_delay_output_action: 120 ``` ### 💱 Convert output to PNG/JPEG or JSON diff --git a/source/plugins/core/metadata.yml b/source/plugins/core/metadata.yml index f3e0046f..a06e58a9 100644 --- a/source/plugins/core/metadata.yml +++ b/source/plugins/core/metadata.yml @@ -244,6 +244,22 @@ inputs: min: 0 max: 3600 + # Number of retries in case output action fail + retries_output_action: + description: Number of retries (output action) + type: number + default: 5 + min: 1 + max: 10 + + # Time to wait (in seconds) before each retry (output action) + retries_delay_output_action: + description: Time to wait (in seconds) before each retry (output action) + type: number + default: 120 + min: 0 + max: 3600 + # ==================================================================================== # 🚧 Options below are mostly used for testing