feat(action): add retries_output_action and retries_delay_output_action (#736)

source/app/action/index.mjs

@@ -42,6 +42,28 @@ async function wait(seconds) {
   await new Promise(solve => setTimeout(solve, seconds * 1000))
 }
 
+//Retry wrapper
+async function retry(func, {retries = 1, delay = 0} = {}) {
+  let error = null
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      console.debug(`::group::Attempt ${attempt}/${retries}`)
+      const result = await func()
+      console.debug("::endgroup::")
+      return result
+    }
+    catch (_error) {
+      error = _error
+      console.debug("::endgroup::")
+      console.debug(`::warning::${error.message}`)
+      await wait(delay)
+    }
+  }
+  if (error)
+    throw error
+  return null
+}
+
 //Runner
 (async function() {
   try {
@@ -92,6 +114,8 @@ async function wait(seconds) {
       "use.prebuilt.image":_image,
       retries,
       "retries.delay":retries_delay,
+      "retries.output.action":retries_output_action,
+      "retries.delay.output.action":retries_delay_output_action,
       "output.action":_action,
       "output.condition":_output_condition,
       delay,
@@ -303,23 +327,12 @@ async function wait(seconds) {
     //Render metrics
     info.break()
     info.section("Rendering")
-    let error = null, rendered = null
+    let rendered = await retry(async () => {
-    for (let attempt = 1; attempt <= retries; attempt++) {
+      const {rendered} = await metrics({login:user, q}, {graphql, rest, plugins, conf, die, verify, convert}, {Plugins, Templates})
-      try {
+      return rendered
-        console.debug(`::group::Attempt ${attempt}/${retries}`)
+    }, {retries, delay:retries_delay})
-        ;({rendered} = await metrics({login:user, q}, {graphql, rest, plugins, conf, die, verify, convert}, {Plugins, Templates}))
-        console.debug("::endgroup::")
-        break
-      }
-      catch (_error) {
-        error = _error
-        console.debug("::endgroup::")
-        console.debug(`::warning::rendering failed (${error.message})`)
-        await wait(retries_delay)
-      }
-    }
     if (!rendered)
-      throw error ?? new Error("Could not render metrics")
+      throw new Error("Could not render metrics")
     info("Status", "complete")
 
     //Output condition
@@ -329,6 +342,7 @@ async function wait(seconds) {
     if ((_output_condition === "data-changed")&&((committer.commit) || (committer.pr))) {
       const {svg} = await import("../metrics/utils.mjs")
       let data = ""
+      await retry(async () => {
         try {
           data = `${Buffer.from((await committer.rest.repos.getContent({...github.context.repo, ref:`heads/${committer.head}`, path:filename})).data.content, "base64")}`
         }
@@ -336,6 +350,7 @@ async function wait(seconds) {
           if (error.response.status !== 404)
             throw error
         }
+      }, {retries:retries_output_action, delay:retries_delay_output_action})
       const previous = await svg.hash(data)
       info("Previous hash", previous)
       const current = await svg.hash(rendered)
@@ -363,11 +378,12 @@ async function wait(seconds) {
       process.exit(0)
     }
 
-    //Cache
+    //Cache embed svg for markdown outputs
     if (/markdown/.test(convert)) {
       const regex = /(?<match><img class="metrics-cachable" data-name="(?<name>[\s\S]+?)" src="data:image[/](?<format>(?:svg[+]xml)|jpeg|png);base64,(?<content>[/+=\w]+?)">)/
       let matched = null
       while (matched = regex.exec(rendered)?.groups) { //eslint-disable-line no-cond-assign
+        await retry(async () => {
           const {match, name, format, content} = matched
           let path = `${_markdown_cache}/${name}.${format.replace(/[+].*$/g, "")}`
           console.debug(`Processing ${path}`)
@@ -400,6 +416,7 @@ async function wait(seconds) {
             rendered = rendered.replace(match, `<img src="https://github.com/${github.context.repo.owner}/${github.context.repo.repo}/blob/${committer.branch}/${path}">`)
             info(`Saving ${path}`, "ok")
           }
+        }, {retries:retries_output_action, delay:retries_delay_output_action})
       }
     }
 
@@ -416,13 +433,16 @@ async function wait(seconds) {
 
     //Upload to gist (this is done as user since committer_token may not have gist rights)
     if (committer.gist) {
+      await retry(async () => {
         await rest.gists.update({gist_id:committer.gist, files:{[filename]:{content:rendered}}})
         info(`Upload to gist ${committer.gist}`, "ok")
         committer.commit = false
+      }, {retries:retries_output_action, delay:retries_delay_output_action})
     }
 
     //Commit metrics
     if (committer.commit) {
+      await retry(async () => {
         await committer.rest.repos.createOrUpdateFileContents({
           ...github.context.repo,
           path:filename,
@@ -432,12 +452,14 @@ async function wait(seconds) {
           ...(committer.sha ? {sha:committer.sha} : {}),
         })
         info(`Commit to branch ${committer.branch}`, "ok")
+      }, {retries:retries_output_action, delay:retries_delay_output_action})
     }
 
     //Pull request
     if (committer.pr) {
       //Create pull request
       let number = null
+      await retry(async () => {
         try {
           ({data:{number}} = await committer.rest.pulls.create({...github.context.repo, head:committer.head, base:committer.branch, title:`Auto-generated metrics for run #${github.context.runId}`, body:" ", maintainer_can_modify:true}))
           info(`Pull request from ${committer.head} to ${committer.branch}`, "(created)")
@@ -463,27 +485,33 @@ async function wait(seconds) {
           }
           else
             throw error
-
         }
         info("Pull request number", number)
+      }, {retries:retries_output_action, delay:retries_delay_output_action})
       //Merge pull request
       if (committer.merge) {
         info("Merge method", committer.merge)
         let attempts = 240
         do {
+          const success = await retry(async () => {
             //Check pull request mergeability (https://octokit.github.io/rest.js/v18#pulls-get)
             const {data:{mergeable, mergeable_state:state}} = await committer.rest.pulls.get({...github.context.repo, pull_number:number})
             console.debug(`Pull request #${number} mergeable state is "${state}"`)
             if (mergeable === null) {
               await wait(15)
-            continue
+              return false
             }
             if (!mergeable)
               throw new Error(`Pull request #${number} is not mergeable (state is "${state}")`)
             //Merge pull request
             await committer.rest.pulls.merge({...github.context.repo, pull_number:number, merge_method:committer.merge})
             info(`Merge #${number} to ${committer.branch}`, "ok")
+            return true
+          }, {retries:retries_output_action, delay:retries_delay_output_action})
+          if (!success)
+            continue
           //Delete head branch
+          await retry(async () => {
             try {
               await wait(15)
               await committer.rest.git.deleteRef({...github.context.repo, ref:`heads/${committer.head}`})
@@ -494,6 +522,7 @@ async function wait(seconds) {
                 throw error
             }
             info(`Branch ${committer.head}`, "(deleted)")
+          }, {retries:retries_output_action, delay:retries_delay_output_action})
           break
         } while (--attempts)
       }

source/plugins/core/README.md

@@ -228,11 +228,14 @@ It also possible to alter output condition using `output_condition` option, whic
     output_action: pull-request-merge
 ```
 
-### ♻️ Retrying automatically failed rendering
+### ♻️ Retrying automatically failed rendering and output action
 
 Rendering is subject to external factors and can fail from time to time.
 It is possible to mitigate this issue using `retries` and `retries_delay` options to automatically retry later metrics rendering and avoid workflow fails.
 
+Output action is also subject to GitHub API rate-limiting and status and can fail from time to time.
+It is possible to mitigate this issue using `retries_output_action` and `retries_delay_output_action` options to automatically retry later metrics output action and avoid workflow fails. As this is a separate step from rendering, metrics rendering won't be computed again during this phase.
+
 #### ℹ️ Examples workflows
 
 ```yaml
@@ -241,6 +244,8 @@ It is possible to mitigate this issue using `retries` and `retries_delay` option
     # ... other options
     retries: 3
     retries_delay: 300
+    retries_output_action: 5
+    retries_delay_output_action: 120
 ```
 
 ### 💱 Convert output to PNG/JPEG or JSON

source/plugins/core/metadata.yml

@@ -244,6 +244,22 @@ inputs:
     min: 0
     max: 3600
 
+  # Number of retries in case output action fail
+  retries_output_action:
+    description: Number of retries (output action)
+    type: number
+    default: 5
+    min: 1
+    max: 10
+
+  # Time to wait (in seconds) before each retry (output action)
+  retries_delay_output_action:
+    description: Time to wait (in seconds) before each retry (output action)
+    type: number
+    default: 120
+    min: 0
+    max: 3600
+
   # ====================================================================================
   # 🚧 Options below are mostly used for testing