feat(action): add retries_output_action and retries_delay_output_action (#736)

source/app/action/index.mjs

@@ -42,6 +42,28 @@ async function wait(seconds) {
   await new Promise(solve => setTimeout(solve, seconds * 1000))
 }
 
+//Retry wrapper
+async function retry(func, {retries = 1, delay = 0} = {}) {
+  let error = null
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      console.debug(`::group::Attempt ${attempt}/${retries}`)
+      const result = await func()
+      console.debug("::endgroup::")
+      return result
+    }
+    catch (_error) {
+      error = _error
+      console.debug("::endgroup::")
+      console.debug(`::warning::${error.message}`)
+      await wait(delay)
+    }
+  }
+  if (error)
+    throw error
+  return null
+}
+
 //Runner
 (async function() {
   try {
@@ -92,6 +114,8 @@ async function wait(seconds) {
       "use.prebuilt.image":_image,
       retries,
       "retries.delay":retries_delay,
+      "retries.output.action":retries_output_action,
+      "retries.delay.output.action":retries_delay_output_action,
       "output.action":_action,
       "output.condition":_output_condition,
       delay,
@@ -303,23 +327,12 @@ async function wait(seconds) {
     //Render metrics
     info.break()
     info.section("Rendering")
-    let error = null, rendered = null
+    let rendered = await retry(async () => {
-    for (let attempt = 1; attempt <= retries; attempt++) {
+      const {rendered} = await metrics({login:user, q}, {graphql, rest, plugins, conf, die, verify, convert}, {Plugins, Templates})
-      try {
+      return rendered
-        console.debug(`::group::Attempt ${attempt}/${retries}`)
+    }, {retries, delay:retries_delay})
-        ;({rendered} = await metrics({login:user, q}, {graphql, rest, plugins, conf, die, verify, convert}, {Plugins, Templates}))
-        console.debug("::endgroup::")
-        break
-      }
-      catch (_error) {
-        error = _error
-        console.debug("::endgroup::")
-        console.debug(`::warning::rendering failed (${error.message})`)
-        await wait(retries_delay)
-      }
-    }
     if (!rendered)
-      throw error ?? new Error("Could not render metrics")
+      throw new Error("Could not render metrics")
     info("Status", "complete")
 
     //Output condition
@@ -329,13 +342,15 @@ async function wait(seconds) {
     if ((_output_condition === "data-changed")&&((committer.commit) || (committer.pr))) {
       const {svg} = await import("../metrics/utils.mjs")
       let data = ""
-      try {
+      await retry(async () => {
-        data = `${Buffer.from((await committer.rest.repos.getContent({...github.context.repo, ref:`heads/${committer.head}`, path:filename})).data.content, "base64")}`
+        try {
-      }
+          data = `${Buffer.from((await committer.rest.repos.getContent({...github.context.repo, ref:`heads/${committer.head}`, path:filename})).data.content, "base64")}`
-      catch (error) {
+        }
-        if (error.response.status !== 404)
+        catch (error) {
-          throw error
+          if (error.response.status !== 404)
-      }
+            throw error
+        }
+      }, {retries:retries_output_action, delay:retries_delay_output_action})
       const previous = await svg.hash(data)
       info("Previous hash", previous)
       const current = await svg.hash(rendered)
@@ -363,43 +378,45 @@ async function wait(seconds) {
       process.exit(0)
     }
 
-    //Cache
+    //Cache embed svg for markdown outputs
     if (/markdown/.test(convert)) {
       const regex = /(?<match><img class="metrics-cachable" data-name="(?<name>[\s\S]+?)" src="data:image[/](?<format>(?:svg[+]xml)|jpeg|png);base64,(?<content>[/+=\w]+?)">)/
       let matched = null
       while (matched = regex.exec(rendered)?.groups) { //eslint-disable-line no-cond-assign
-        const {match, name, format, content} = matched
+        await retry(async () => {
-        let path = `${_markdown_cache}/${name}.${format.replace(/[+].*$/g, "")}`
+          const {match, name, format, content} = matched
-        console.debug(`Processing ${path}`)
+          let path = `${_markdown_cache}/${name}.${format.replace(/[+].*$/g, "")}`
-        let sha = null
+          console.debug(`Processing ${path}`)
-        try {
+          let sha = null
-          const {repository:{object:{oid}}} = await graphql(
+          try {
-            `
+            const {repository:{object:{oid}}} = await graphql(
-              query Sha {
+              `
-                repository(owner: "${github.context.repo.owner}", name: "${github.context.repo.repo}") {
+                query Sha {
-                  object(expression: "${committer.head}:${path}") { ... on Blob { oid } }
+                  repository(owner: "${github.context.repo.owner}", name: "${github.context.repo.repo}") {
+                    object(expression: "${committer.head}:${path}") { ... on Blob { oid } }
+                  }
                 }
-              }
+              `,
-            `,
+              {headers:{authorization:`token ${committer.token}`}},
-            {headers:{authorization:`token ${committer.token}`}},
+            )
-          )
+            sha = oid
-          sha = oid
+          }
-        }
+          catch (error) {
-        catch (error) {
+            console.debug(error)
-          console.debug(error)
+          }
-        }
+          finally {
-        finally {
+            await committer.rest.repos.createOrUpdateFileContents({
-          await committer.rest.repos.createOrUpdateFileContents({
+              ...github.context.repo,
-            ...github.context.repo,
+              path,
-            path,
+              content,
-            content,
+              message:`${committer.message} (cache)`,
-            message:`${committer.message} (cache)`,
+              ...(sha ? {sha} : {}),
-            ...(sha ? {sha} : {}),
+              branch:committer.pr ? committer.head : committer.branch,
-            branch:committer.pr ? committer.head : committer.branch,
+            })
-          })
+            rendered = rendered.replace(match, `<img src="https://github.com/${github.context.repo.owner}/${github.context.repo.repo}/blob/${committer.branch}/${path}">`)
-          rendered = rendered.replace(match, `<img src="https://github.com/${github.context.repo.owner}/${github.context.repo.repo}/blob/${committer.branch}/${path}">`)
+            info(`Saving ${path}`, "ok")
-          info(`Saving ${path}`, "ok")
+          }
-        }
+        }, {retries:retries_output_action, delay:retries_delay_output_action})
       }
     }
 
@@ -416,84 +433,96 @@ async function wait(seconds) {
 
     //Upload to gist (this is done as user since committer_token may not have gist rights)
     if (committer.gist) {
-      await rest.gists.update({gist_id:committer.gist, files:{[filename]:{content:rendered}}})
+      await retry(async () => {
-      info(`Upload to gist ${committer.gist}`, "ok")
+        await rest.gists.update({gist_id:committer.gist, files:{[filename]:{content:rendered}}})
-      committer.commit = false
+        info(`Upload to gist ${committer.gist}`, "ok")
+        committer.commit = false
+      }, {retries:retries_output_action, delay:retries_delay_output_action})
     }
 
     //Commit metrics
     if (committer.commit) {
-      await committer.rest.repos.createOrUpdateFileContents({
+      await retry(async () => {
-        ...github.context.repo,
+        await committer.rest.repos.createOrUpdateFileContents({
-        path:filename,
+          ...github.context.repo,
-        message:committer.message,
+          path:filename,
-        content:Buffer.from(typeof rendered === "object" ? JSON.stringify(rendered) : `${rendered}`).toString("base64"),
+          message:committer.message,
-        branch:committer.pr ? committer.head : committer.branch,
+          content:Buffer.from(typeof rendered === "object" ? JSON.stringify(rendered) : `${rendered}`).toString("base64"),
-        ...(committer.sha ? {sha:committer.sha} : {}),
+          branch:committer.pr ? committer.head : committer.branch,
-      })
+          ...(committer.sha ? {sha:committer.sha} : {}),
-      info(`Commit to branch ${committer.branch}`, "ok")
+        })
+        info(`Commit to branch ${committer.branch}`, "ok")
+      }, {retries:retries_output_action, delay:retries_delay_output_action})
     }
 
     //Pull request
     if (committer.pr) {
       //Create pull request
       let number = null
-      try {
+      await retry(async () => {
-        ({data:{number}} = await committer.rest.pulls.create({...github.context.repo, head:committer.head, base:committer.branch, title:`Auto-generated metrics for run #${github.context.runId}`, body:" ", maintainer_can_modify:true}))
+        try {
-        info(`Pull request from ${committer.head} to ${committer.branch}`, "(created)")
+          ({data:{number}} = await committer.rest.pulls.create({...github.context.repo, head:committer.head, base:committer.branch, title:`Auto-generated metrics for run #${github.context.runId}`, body:" ", maintainer_can_modify:true}))
-      }
+          info(`Pull request from ${committer.head} to ${committer.branch}`, "(created)")
-      catch (error) {
-        console.debug(error)
-        //Check if pull request has already been created previously
-        if (/A pull request already exists/.test(error)) {
-          info(`Pull request from ${committer.head} to ${committer.branch}`, "(already existing)")
-          const q = `repo:${github.context.repo.owner}/${github.context.repo.repo}+type:pr+state:open+Auto-generated metrics for run #${github.context.runId}+in:title`
-          const prs = (await committer.rest.search.issuesAndPullRequests({q})).data.items.filter(({user:{login}}) => login === "github-actions[bot]")
-          if (prs.length < 1)
-            throw new Error("0 matching prs. Cannot proceed.")
-          if (prs.length > 1)
-            throw new Error(`Found more than one matching prs: ${prs.map(({number}) => `#${number}`).join(", ")}. Cannot proceed.`)
-          ;({number} = prs.shift())
         }
-        //Check if pull request could not been created because there are no diff between head and base
+        catch (error) {
-        else if (/No commits between/.test(error)) {
+          console.debug(error)
-          info(`Pull request from ${committer.head} to ${committer.branch}`, "(no diff)")
+          //Check if pull request has already been created previously
-          committer.merge = false
+          if (/A pull request already exists/.test(error)) {
-          number = "(none)"
+            info(`Pull request from ${committer.head} to ${committer.branch}`, "(already existing)")
+            const q = `repo:${github.context.repo.owner}/${github.context.repo.repo}+type:pr+state:open+Auto-generated metrics for run #${github.context.runId}+in:title`
+            const prs = (await committer.rest.search.issuesAndPullRequests({q})).data.items.filter(({user:{login}}) => login === "github-actions[bot]")
+            if (prs.length < 1)
+              throw new Error("0 matching prs. Cannot proceed.")
+            if (prs.length > 1)
+              throw new Error(`Found more than one matching prs: ${prs.map(({number}) => `#${number}`).join(", ")}. Cannot proceed.`)
+            ;({number} = prs.shift())
+          }
+          //Check if pull request could not been created because there are no diff between head and base
+          else if (/No commits between/.test(error)) {
+            info(`Pull request from ${committer.head} to ${committer.branch}`, "(no diff)")
+            committer.merge = false
+            number = "(none)"
+          }
+          else
+            throw error
         }
-        else
+        info("Pull request number", number)
-          throw error
+      }, {retries:retries_output_action, delay:retries_delay_output_action})
-
-      }
-      info("Pull request number", number)
       //Merge pull request
       if (committer.merge) {
         info("Merge method", committer.merge)
         let attempts = 240
         do {
-          //Check pull request mergeability (https://octokit.github.io/rest.js/v18#pulls-get)
+          const success = await retry(async () => {
-          const {data:{mergeable, mergeable_state:state}} = await committer.rest.pulls.get({...github.context.repo, pull_number:number})
+            //Check pull request mergeability (https://octokit.github.io/rest.js/v18#pulls-get)
-          console.debug(`Pull request #${number} mergeable state is "${state}"`)
+            const {data:{mergeable, mergeable_state:state}} = await committer.rest.pulls.get({...github.context.repo, pull_number:number})
-          if (mergeable === null) {
+            console.debug(`Pull request #${number} mergeable state is "${state}"`)
-            await wait(15)
+            if (mergeable === null) {
+              await wait(15)
+              return false
+            }
+            if (!mergeable)
+              throw new Error(`Pull request #${number} is not mergeable (state is "${state}")`)
+            //Merge pull request
+            await committer.rest.pulls.merge({...github.context.repo, pull_number:number, merge_method:committer.merge})
+            info(`Merge #${number} to ${committer.branch}`, "ok")
+            return true
+          }, {retries:retries_output_action, delay:retries_delay_output_action})
+          if (!success)
             continue
-          }
-          if (!mergeable)
-            throw new Error(`Pull request #${number} is not mergeable (state is "${state}")`)
-          //Merge pull request
-          await committer.rest.pulls.merge({...github.context.repo, pull_number:number, merge_method:committer.merge})
-          info(`Merge #${number} to ${committer.branch}`, "ok")
           //Delete head branch
-          try {
+          await retry(async () => {
-            await wait(15)
+            try {
-            await committer.rest.git.deleteRef({...github.context.repo, ref:`heads/${committer.head}`})
+              await wait(15)
-          }
+              await committer.rest.git.deleteRef({...github.context.repo, ref:`heads/${committer.head}`})
-          catch (error) {
+            }
-            console.debug(error)
+            catch (error) {
-            if (!/reference does not exist/i.test(`${error}`))
+              console.debug(error)
-              throw error
+              if (!/reference does not exist/i.test(`${error}`))
-          }
+                throw error
-          info(`Branch ${committer.head}`, "(deleted)")
+            }
+            info(`Branch ${committer.head}`, "(deleted)")
+          }, {retries:retries_output_action, delay:retries_delay_output_action})
           break
         } while (--attempts)
       }

source/plugins/core/README.md

@@ -228,11 +228,14 @@ It also possible to alter output condition using `output_condition` option, whic
     output_action: pull-request-merge
 ```
 
-### ♻️ Retrying automatically failed rendering
+### ♻️ Retrying automatically failed rendering and output action
 
 Rendering is subject to external factors and can fail from time to time.
 It is possible to mitigate this issue using `retries` and `retries_delay` options to automatically retry later metrics rendering and avoid workflow fails.
 
+Output action is also subject to GitHub API rate-limiting and status and can fail from time to time.
+It is possible to mitigate this issue using `retries_output_action` and `retries_delay_output_action` options to automatically retry later metrics output action and avoid workflow fails. As this is a separate step from rendering, metrics rendering won't be computed again during this phase.
+
 #### ℹ️ Examples workflows
 
 ```yaml
@@ -241,6 +244,8 @@ It is possible to mitigate this issue using `retries` and `retries_delay` option
     # ... other options
     retries: 3
     retries_delay: 300
+    retries_output_action: 5
+    retries_delay_output_action: 120
 ```
 
 ### 💱 Convert output to PNG/JPEG or JSON

source/plugins/core/metadata.yml

@@ -244,6 +244,22 @@ inputs:
     min: 0
     max: 3600
 
+  # Number of retries in case output action fail
+  retries_output_action:
+    description: Number of retries (output action)
+    type: number
+    default: 5
+    min: 1
+    max: 10
+
+  # Time to wait (in seconds) before each retry (output action)
+  retries_delay_output_action:
+    description: Time to wait (in seconds) before each retry (output action)
+    type: number
+    default: 120
+    min: 0
+    max: 3600
+
   # ====================================================================================
   # 🚧 Options below are mostly used for testing