feat(action): add retries_output_action and retries_delay_output_action (#736)

2022-01-02 16:10:03 +01:00
parent b85f2f3178
commit edb67f4168
3 changed files with 165 additions and 115 deletions
--- a/source/app/action/index.mjs
+++ b/source/app/action/index.mjs
@@ -42,6 +42,28 @@ async function wait(seconds) {
  await new Promise(solve => setTimeout(solve, seconds * 1000))
 }

+//Retry wrapper
+async function retry(func, {retries = 1, delay = 0} = {}) {
+  let error = null
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      console.debug(`::group::Attempt ${attempt}/${retries}`)
+      const result = await func()
+      console.debug("::endgroup::")
+      return result
+    }
+    catch (_error) {
+      error = _error
+      console.debug("::endgroup::")
+      console.debug(`::warning::${error.message}`)
+      await wait(delay)
+    }
+  }
+  if (error)
+    throw error
+  return null
+}
+
 //Runner
 (async function() {
  try {
@@ -92,6 +114,8 @@ async function wait(seconds) {
      "use.prebuilt.image":_image,
      retries,
      "retries.delay":retries_delay,
+      "retries.output.action":retries_output_action,
+      "retries.delay.output.action":retries_delay_output_action,
      "output.action":_action,
      "output.condition":_output_condition,
      delay,
@@ -303,23 +327,12 @@ async function wait(seconds) {
    //Render metrics
    info.break()
    info.section("Rendering")
-    let error = null, rendered = null
-    for (let attempt = 1; attempt <= retries; attempt++) {
-      try {
-        console.debug(`::group::Attempt ${attempt}/${retries}`)
-        ;({rendered} = await metrics({login:user, q}, {graphql, rest, plugins, conf, die, verify, convert}, {Plugins, Templates}))
-        console.debug("::endgroup::")
-        break
-      }
-      catch (_error) {
-        error = _error
-        console.debug("::endgroup::")
-        console.debug(`::warning::rendering failed (${error.message})`)
-        await wait(retries_delay)
-      }
-    }
+    let rendered = await retry(async () => {
+      const {rendered} = await metrics({login:user, q}, {graphql, rest, plugins, conf, die, verify, convert}, {Plugins, Templates})
+      return rendered
+    }, {retries, delay:retries_delay})
    if (!rendered)
-      throw error ?? new Error("Could not render metrics")
+      throw new Error("Could not render metrics")
    info("Status", "complete")

    //Output condition
@@ -329,13 +342,15 @@ async function wait(seconds) {
    if ((_output_condition === "data-changed")&&((committer.commit) || (committer.pr))) {
      const {svg} = await import("../metrics/utils.mjs")
      let data = ""
-      try {
-        data = `${Buffer.from((await committer.rest.repos.getContent({...github.context.repo, ref:`heads/${committer.head}`, path:filename})).data.content, "base64")}`
-      }
-      catch (error) {
-        if (error.response.status !== 404)
-          throw error
-      }
+      await retry(async () => {
+        try {
+          data = `${Buffer.from((await committer.rest.repos.getContent({...github.context.repo, ref:`heads/${committer.head}`, path:filename})).data.content, "base64")}`
+        }
+        catch (error) {
+          if (error.response.status !== 404)
+            throw error
+        }
+      }, {retries:retries_output_action, delay:retries_delay_output_action})
      const previous = await svg.hash(data)
      info("Previous hash", previous)
      const current = await svg.hash(rendered)
@@ -363,43 +378,45 @@ async function wait(seconds) {
      process.exit(0)
    }

-    //Cache
+    //Cache embed svg for markdown outputs
    if (/markdown/.test(convert)) {
      const regex = /(?<match><img class="metrics-cachable" data-name="(?<name>[\s\S]+?)" src="data:image[/](?<format>(?:svg[+]xml)|jpeg|png);base64,(?<content>[/+=\w]+?)">)/
      let matched = null
      while (matched = regex.exec(rendered)?.groups) { //eslint-disable-line no-cond-assign
-        const {match, name, format, content} = matched
-        let path = `${_markdown_cache}/${name}.${format.replace(/[+].*$/g, "")}`
-        console.debug(`Processing ${path}`)
-        let sha = null
-        try {
-          const {repository:{object:{oid}}} = await graphql(
-            `
-              query Sha {
-                repository(owner: "${github.context.repo.owner}", name: "${github.context.repo.repo}") {
-                  object(expression: "${committer.head}:${path}") { ... on Blob { oid } }
+        await retry(async () => {
+          const {match, name, format, content} = matched
+          let path = `${_markdown_cache}/${name}.${format.replace(/[+].*$/g, "")}`
+          console.debug(`Processing ${path}`)
+          let sha = null
+          try {
+            const {repository:{object:{oid}}} = await graphql(
+              `
+                query Sha {
+                  repository(owner: "${github.context.repo.owner}", name: "${github.context.repo.repo}") {
+                    object(expression: "${committer.head}:${path}") { ... on Blob { oid } }
+                  }
                }
-              }
-            `,
-            {headers:{authorization:`token ${committer.token}`}},
-          )
-          sha = oid
-        }
-        catch (error) {
-          console.debug(error)
-        }
-        finally {
-          await committer.rest.repos.createOrUpdateFileContents({
-            ...github.context.repo,
-            path,
-            content,
-            message:`${committer.message} (cache)`,
-            ...(sha ? {sha} : {}),
-            branch:committer.pr ? committer.head : committer.branch,
-          })
-          rendered = rendered.replace(match, `<img src="https://github.com/${github.context.repo.owner}/${github.context.repo.repo}/blob/${committer.branch}/${path}">`)
-          info(`Saving ${path}`, "ok")
-        }
+              `,
+              {headers:{authorization:`token ${committer.token}`}},
+            )
+            sha = oid
+          }
+          catch (error) {
+            console.debug(error)
+          }
+          finally {
+            await committer.rest.repos.createOrUpdateFileContents({
+              ...github.context.repo,
+              path,
+              content,
+              message:`${committer.message} (cache)`,
+              ...(sha ? {sha} : {}),
+              branch:committer.pr ? committer.head : committer.branch,
+            })
+            rendered = rendered.replace(match, `<img src="https://github.com/${github.context.repo.owner}/${github.context.repo.repo}/blob/${committer.branch}/${path}">`)
+            info(`Saving ${path}`, "ok")
+          }
+        }, {retries:retries_output_action, delay:retries_delay_output_action})
      }
    }

@@ -416,84 +433,96 @@ async function wait(seconds) {

    //Upload to gist (this is done as user since committer_token may not have gist rights)
    if (committer.gist) {
-      await rest.gists.update({gist_id:committer.gist, files:{[filename]:{content:rendered}}})
-      info(`Upload to gist ${committer.gist}`, "ok")
-      committer.commit = false
+      await retry(async () => {
+        await rest.gists.update({gist_id:committer.gist, files:{[filename]:{content:rendered}}})
+        info(`Upload to gist ${committer.gist}`, "ok")
+        committer.commit = false
+      }, {retries:retries_output_action, delay:retries_delay_output_action})
    }

    //Commit metrics
    if (committer.commit) {
-      await committer.rest.repos.createOrUpdateFileContents({
-        ...github.context.repo,
-        path:filename,
-        message:committer.message,
-        content:Buffer.from(typeof rendered === "object" ? JSON.stringify(rendered) : `${rendered}`).toString("base64"),
-        branch:committer.pr ? committer.head : committer.branch,
-        ...(committer.sha ? {sha:committer.sha} : {}),
-      })
-      info(`Commit to branch ${committer.branch}`, "ok")
+      await retry(async () => {
+        await committer.rest.repos.createOrUpdateFileContents({
+          ...github.context.repo,
+          path:filename,
+          message:committer.message,
+          content:Buffer.from(typeof rendered === "object" ? JSON.stringify(rendered) : `${rendered}`).toString("base64"),
+          branch:committer.pr ? committer.head : committer.branch,
+          ...(committer.sha ? {sha:committer.sha} : {}),
+        })
+        info(`Commit to branch ${committer.branch}`, "ok")
+      }, {retries:retries_output_action, delay:retries_delay_output_action})
    }

    //Pull request
    if (committer.pr) {
      //Create pull request
      let number = null
-      try {
-        ({data:{number}} = await committer.rest.pulls.create({...github.context.repo, head:committer.head, base:committer.branch, title:`Auto-generated metrics for run #${github.context.runId}`, body:" ", maintainer_can_modify:true}))
-        info(`Pull request from ${committer.head} to ${committer.branch}`, "(created)")
-      }
-      catch (error) {
-        console.debug(error)
-        //Check if pull request has already been created previously
-        if (/A pull request already exists/.test(error)) {
-          info(`Pull request from ${committer.head} to ${committer.branch}`, "(already existing)")
-          const q = `repo:${github.context.repo.owner}/${github.context.repo.repo}+type:pr+state:open+Auto-generated metrics for run #${github.context.runId}+in:title`
-          const prs = (await committer.rest.search.issuesAndPullRequests({q})).data.items.filter(({user:{login}}) => login === "github-actions[bot]")
-          if (prs.length < 1)
-            throw new Error("0 matching prs. Cannot proceed.")
-          if (prs.length > 1)
-            throw new Error(`Found more than one matching prs: ${prs.map(({number}) => `#${number}`).join(", ")}. Cannot proceed.`)
-          ;({number} = prs.shift())
+      await retry(async () => {
+        try {
+          ({data:{number}} = await committer.rest.pulls.create({...github.context.repo, head:committer.head, base:committer.branch, title:`Auto-generated metrics for run #${github.context.runId}`, body:" ", maintainer_can_modify:true}))
+          info(`Pull request from ${committer.head} to ${committer.branch}`, "(created)")
        }
-        //Check if pull request could not been created because there are no diff between head and base
-        else if (/No commits between/.test(error)) {
-          info(`Pull request from ${committer.head} to ${committer.branch}`, "(no diff)")
-          committer.merge = false
-          number = "(none)"
+        catch (error) {
+          console.debug(error)
+          //Check if pull request has already been created previously
+          if (/A pull request already exists/.test(error)) {
+            info(`Pull request from ${committer.head} to ${committer.branch}`, "(already existing)")
+            const q = `repo:${github.context.repo.owner}/${github.context.repo.repo}+type:pr+state:open+Auto-generated metrics for run #${github.context.runId}+in:title`
+            const prs = (await committer.rest.search.issuesAndPullRequests({q})).data.items.filter(({user:{login}}) => login === "github-actions[bot]")
+            if (prs.length < 1)
+              throw new Error("0 matching prs. Cannot proceed.")
+            if (prs.length > 1)
+              throw new Error(`Found more than one matching prs: ${prs.map(({number}) => `#${number}`).join(", ")}. Cannot proceed.`)
+            ;({number} = prs.shift())
+          }
+          //Check if pull request could not been created because there are no diff between head and base
+          else if (/No commits between/.test(error)) {
+            info(`Pull request from ${committer.head} to ${committer.branch}`, "(no diff)")
+            committer.merge = false
+            number = "(none)"
+          }
+          else
+            throw error
        }
-        else
-          throw error
-
-      }
-      info("Pull request number", number)
+        info("Pull request number", number)
+      }, {retries:retries_output_action, delay:retries_delay_output_action})
      //Merge pull request
      if (committer.merge) {
        info("Merge method", committer.merge)
        let attempts = 240
        do {
-          //Check pull request mergeability (https://octokit.github.io/rest.js/v18#pulls-get)
-          const {data:{mergeable, mergeable_state:state}} = await committer.rest.pulls.get({...github.context.repo, pull_number:number})
-          console.debug(`Pull request #${number} mergeable state is "${state}"`)
-          if (mergeable === null) {
-            await wait(15)
+          const success = await retry(async () => {
+            //Check pull request mergeability (https://octokit.github.io/rest.js/v18#pulls-get)
+            const {data:{mergeable, mergeable_state:state}} = await committer.rest.pulls.get({...github.context.repo, pull_number:number})
+            console.debug(`Pull request #${number} mergeable state is "${state}"`)
+            if (mergeable === null) {
+              await wait(15)
+              return false
+            }
+            if (!mergeable)
+              throw new Error(`Pull request #${number} is not mergeable (state is "${state}")`)
+            //Merge pull request
+            await committer.rest.pulls.merge({...github.context.repo, pull_number:number, merge_method:committer.merge})
+            info(`Merge #${number} to ${committer.branch}`, "ok")
+            return true
+          }, {retries:retries_output_action, delay:retries_delay_output_action})
+          if (!success)
            continue
-          }
-          if (!mergeable)
-            throw new Error(`Pull request #${number} is not mergeable (state is "${state}")`)
-          //Merge pull request
-          await committer.rest.pulls.merge({...github.context.repo, pull_number:number, merge_method:committer.merge})
-          info(`Merge #${number} to ${committer.branch}`, "ok")
          //Delete head branch
-          try {
-            await wait(15)
-            await committer.rest.git.deleteRef({...github.context.repo, ref:`heads/${committer.head}`})
-          }
-          catch (error) {
-            console.debug(error)
-            if (!/reference does not exist/i.test(`${error}`))
-              throw error
-          }
-          info(`Branch ${committer.head}`, "(deleted)")
+          await retry(async () => {
+            try {
+              await wait(15)
+              await committer.rest.git.deleteRef({...github.context.repo, ref:`heads/${committer.head}`})
+            }
+            catch (error) {
+              console.debug(error)
+              if (!/reference does not exist/i.test(`${error}`))
+                throw error
+            }
+            info(`Branch ${committer.head}`, "(deleted)")
+          }, {retries:retries_output_action, delay:retries_delay_output_action})
          break
        } while (--attempts)
      }
--- a/source/plugins/core/README.md
+++ b/source/plugins/core/README.md
@@ -228,11 +228,14 @@ It also possible to alter output condition using `output_condition` option, whic
    output_action: pull-request-merge
 ```

-### ♻️ Retrying automatically failed rendering
+### ♻️ Retrying automatically failed rendering and output action

 Rendering is subject to external factors and can fail from time to time.
 It is possible to mitigate this issue using `retries` and `retries_delay` options to automatically retry later metrics rendering and avoid workflow fails.

+Output action is also subject to GitHub API rate-limiting and status and can fail from time to time.
+It is possible to mitigate this issue using `retries_output_action` and `retries_delay_output_action` options to automatically retry later metrics output action and avoid workflow fails. As this is a separate step from rendering, metrics rendering won't be computed again during this phase.
+
 #### ℹ️ Examples workflows

 ```yaml
@@ -241,6 +244,8 @@ It is possible to mitigate this issue using `retries` and `retries_delay` option
    # ... other options
    retries: 3
    retries_delay: 300
+    retries_output_action: 5
+    retries_delay_output_action: 120
 ```

 ### 💱 Convert output to PNG/JPEG or JSON
--- a/source/plugins/core/metadata.yml
+++ b/source/plugins/core/metadata.yml
@@ -244,6 +244,22 @@ inputs:
    min: 0
    max: 3600

+  # Number of retries in case output action fail
+  retries_output_action:
+    description: Number of retries (output action)
+    type: number
+    default: 5
+    min: 1
+    max: 10
+
+  # Time to wait (in seconds) before each retry (output action)
+  retries_delay_output_action:
+    description: Time to wait (in seconds) before each retry (output action)
+    type: number
+    default: 120
+    min: 0
+    max: 3600
+
  # ====================================================================================
  # 🚧 Options below are mostly used for testing