Retry all http calls for artifact upload and download

packages/artifact/RELEASES.md

@@ -50,3 +50,7 @@
 
 - Improved retry-ability when a partial artifact download is encountered
 
+### 0.5.0
+
+- Improved retry-ability for all http calls during artifact upload and download if an error is encountered
+

packages/artifact/__tests__/download.test.ts

@@ -71,7 +71,7 @@ describe('Download Tests', () => {
     setupFailedResponse()
     const downloadHttpClient = new DownloadHttpClient()
     expect(downloadHttpClient.listArtifacts()).rejects.toThrow(
-      'Unable to list artifacts for the run'
+      'List Artifacts failed: Artifact service responded with 500'
     )
   })
 
@@ -113,7 +113,7 @@ describe('Download Tests', () => {
         configVariables.getRuntimeUrl()
       )
     ).rejects.toThrow(
-      `Unable to get ContainersItems from ${configVariables.getRuntimeUrl()}`
+      `Get Container Items failed: Artifact service responded with 500`
     )
   })
 

packages/artifact/__tests__/retry.test.ts

@@ -0,0 +1,114 @@
+import * as http from 'http'
+import * as net from 'net'
+import * as core from '@actions/core'
+import * as configVariables from '../src/internal/config-variables'
+import {retry} from '../src/internal/requestUtils'
+import {IHttpClientResponse} from '@actions/http-client/interfaces'
+import {HttpClientResponse} from '@actions/http-client'
+
+jest.mock('../src/internal/config-variables')
+
+interface ITestResult {
+  responseCode: number
+  errorMessage: string | null
+}
+
+async function testRetry(
+  responseCodes: number[],
+  expectedResult: ITestResult
+): Promise<void> {
+  const reverse = responseCodes.reverse() // Reverse responses since we pop from end
+  if (expectedResult.errorMessage) {
+    // we expect some exception to be thrown
+    expect(
+      retry(
+        'test',
+        async () => handleResponse(reverse.pop()),
+        new Map(), // extra error message for any particular http codes
+        configVariables.getRetryLimit()
+      )
+    ).rejects.toThrow(expectedResult.errorMessage)
+  } else {
+    // we expect a correct status code to be returned
+    const actualResult = await retry(
+      'test',
+      async () => handleResponse(reverse.pop()),
+      new Map(), // extra error message for any particular http codes
+      configVariables.getRetryLimit()
+    )
+    expect(actualResult.message.statusCode).toEqual(expectedResult.responseCode)
+  }
+}
+
+async function handleResponse(
+  testResponseCode: number | undefined
+): Promise<IHttpClientResponse> {
+  if (!testResponseCode) {
+    throw new Error(
+      'Test incorrectly set up. reverse.pop() was called too many times so not enough test response codes were supplied'
+    )
+  }
+
+  return setupSingleMockResponse(testResponseCode)
+}
+
+beforeAll(async () => {
+  // mock all output so that there is less noise when running tests
+  jest.spyOn(console, 'log').mockImplementation(() => {})
+  jest.spyOn(core, 'debug').mockImplementation(() => {})
+  jest.spyOn(core, 'info').mockImplementation(() => {})
+  jest.spyOn(core, 'warning').mockImplementation(() => {})
+  jest.spyOn(core, 'error').mockImplementation(() => {})
+})
+
+/**
+ * Helpers used to setup mocking for the HttpClient
+ */
+async function emptyMockReadBody(): Promise<string> {
+  return new Promise(resolve => {
+    resolve()
+  })
+}
+
+async function setupSingleMockResponse(
+  statusCode: number
+): Promise<IHttpClientResponse> {
+  const mockMessage = new http.IncomingMessage(new net.Socket())
+  const mockReadBody = emptyMockReadBody
+  mockMessage.statusCode = statusCode
+  return new Promise<HttpClientResponse>(resolve => {
+    resolve({
+      message: mockMessage,
+      readBody: mockReadBody
+    })
+  })
+}
+
+test('retry works on successful response', async () => {
+  await testRetry([200], {
+    responseCode: 200,
+    errorMessage: null
+  })
+})
+
+test('retry works after retryable status code', async () => {
+  await testRetry([503, 200], {
+    responseCode: 200,
+    errorMessage: null
+  })
+})
+
+test('retry fails after exhausting retries', async () => {
+  // __mocks__/config-variables caps the max retry count in tests to 2
+  await testRetry([503, 503, 200], {
+    responseCode: 200,
+    errorMessage: 'test failed: Artifact service responded with 503'
+  })
+})
+
+test('retry fails after non-retryable status code', async () => {
+  await testRetry([500, 200], {
+    responseCode: 500,
+    errorMessage: 'test failed: Artifact service responded with 500'
+  })
+})

packages/artifact/__tests__/upload.test.ts

@@ -102,7 +102,7 @@ describe('Upload Tests', () => {
       uploadHttpClient.createArtifactInFileContainer(artifactName)
     ).rejects.toEqual(
       new Error(
-        `Unable to create a container for the artifact invalid-artifact-name at ${getArtifactUrl()}`
+        `Create Artifact Container failed: The artifact name invalid-artifact-name is not valid. Request URL ${getArtifactUrl()}`
       )
     )
   })
@@ -125,7 +125,7 @@ describe('Upload Tests', () => {
       uploadHttpClient.createArtifactInFileContainer(artifactName)
     ).rejects.toEqual(
       new Error(
-        'Artifact storage quota has been hit. Unable to upload any new artifacts'
+        'Create Artifact Container failed: Artifact storage quota has been hit. Unable to upload any new artifacts'
       )
     )
   })
@@ -362,7 +362,9 @@ describe('Upload Tests', () => {
     const uploadHttpClient = new UploadHttpClient()
     expect(
       uploadHttpClient.patchArtifactSize(-2, 'my-artifact')
-    ).rejects.toThrow('Unable to finish uploading artifact my-artifact')
+    ).rejects.toThrow(
+      'Finalize artifact upload failed: Artifact service responded with 400'
+    )
   })
 
   /**

packages/artifact/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@actions/artifact",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "preview": true,
   "description": "Actions artifact lib",
   "keywords": [

packages/artifact/src/internal/download-http-client.ts

@@ -11,7 +11,8 @@ import {
   tryGetRetryAfterValueTimeInMilliseconds,
   displayHttpDiagnostics,
   getFileSize,
-  rmFile
+  rmFile,
+  sleep
 } from './utils'
 import {URL} from 'url'
 import {StatusReporter} from './status-reporter'
@@ -22,6 +23,7 @@ import {HttpManager} from './http-manager'
 import {DownloadItem} from './download-specification'
 import {getDownloadFileConcurrency, getRetryLimit} from './config-variables'
 import {IncomingHttpHeaders} from 'http'
+import {retryHttpClientRequest} from './requestUtils'
 
 export class DownloadHttpClient {
   // http manager is used for concurrent connections when downloading multiple files at once
@@ -46,16 +48,11 @@ export class DownloadHttpClient {
     // use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately
     const client = this.downloadHttpManager.getClient(0)
     const headers = getDownloadHeaders('application/json')
-    const response = await client.get(artifactUrl, headers)
-    const body: string = await response.readBody()
-
-    if (isSuccessStatusCode(response.message.statusCode) && body) {
-      return JSON.parse(body)
-    }
-    displayHttpDiagnostics(response)
-    throw new Error(
-      `Unable to list artifacts for the run. Resource Url ${artifactUrl}`
+    const response = await retryHttpClientRequest('List Artifacts', async () =>
+      client.get(artifactUrl, headers)
     )
+    const body: string = await response.readBody()
+    return JSON.parse(body)
   }
 
   /**
@@ -74,14 +71,12 @@ export class DownloadHttpClient {
     // use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately
     const client = this.downloadHttpManager.getClient(0)
     const headers = getDownloadHeaders('application/json')
-    const response = await client.get(resourceUrl.toString(), headers)
+    const response = await retryHttpClientRequest(
+      'Get Container Items',
+      async () => client.get(resourceUrl.toString(), headers)
+    )
     const body: string = await response.readBody()
-
-    if (isSuccessStatusCode(response.message.statusCode) && body) {
-      return JSON.parse(body)
-    }
-    displayHttpDiagnostics(response)
-    throw new Error(`Unable to get ContainersItems from ${resourceUrl}`)
+    return JSON.parse(body)
   }
 
   /**
@@ -188,14 +183,14 @@ export class DownloadHttpClient {
           core.info(
             `Backoff due to too many requests, retry #${retryCount}. Waiting for ${retryAfterValue} milliseconds before continuing the download`
           )
-          await new Promise(resolve => setTimeout(resolve, retryAfterValue))
+          await sleep(retryAfterValue)
         } else {
           // Back off using an exponential value that depends on the retry count
           const backoffTime = getExponentialRetryTimeInMilliseconds(retryCount)
           core.info(
             `Exponential backoff for retry #${retryCount}. Waiting for ${backoffTime} milliseconds before continuing the download`
           )
-          await new Promise(resolve => setTimeout(resolve, backoffTime))
+          await sleep(backoffTime)
         }
         core.info(
           `Finished backoff for retry #${retryCount}, continuing with download`

packages/artifact/src/internal/requestUtils.ts

@@ -0,0 +1,79 @@
+import {IHttpClientResponse} from '@actions/http-client/interfaces'
+import {
+  isRetryableStatusCode,
+  isSuccessStatusCode,
+  sleep,
+  getExponentialRetryTimeInMilliseconds,
+  displayHttpDiagnostics
+} from './utils'
+import * as core from '@actions/core'
+import {getRetryLimit} from './config-variables'
+
+export async function retry(
+  name: string,
+  operation: () => Promise<IHttpClientResponse>,
+  customErrorMessages: Map<number, string>,
+  maxAttempts: number
+): Promise<IHttpClientResponse> {
+  let response: IHttpClientResponse | undefined = undefined
+  let statusCode: number | undefined = undefined
+  let isRetryable = false
+  let errorMessage = ''
+  let customErrorInformation: string | undefined = undefined
+  let attempt = 1
+
+  while (attempt <= maxAttempts) {
+    try {
+      response = await operation()
+      statusCode = response.message.statusCode
+
+      if (isSuccessStatusCode(statusCode)) {
+        return response
+      }
+
+      // Extra error information that we want to display if a particular response code is hit
+      if (statusCode) {
+        customErrorInformation = customErrorMessages.get(statusCode)
+      }
+
+      isRetryable = isRetryableStatusCode(statusCode)
+      errorMessage = `Artifact service responded with ${statusCode}`
+    } catch (error) {
+      isRetryable = true
+      errorMessage = error.message
+    }
+
+    if (!isRetryable) {
+      core.info(`${name} - Error is not retryable`)
+      if (response) {
+        displayHttpDiagnostics(response)
+      }
+      break
+    }
+
+    core.info(
+      `${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`
+    )
+
+    await sleep(getExponentialRetryTimeInMilliseconds(attempt))
+    attempt++
+  }
+
+  if (response) {
+    displayHttpDiagnostics(response)
+  }
+
+  if (customErrorInformation) {
+    throw Error(`${name} failed: ${customErrorInformation}`)
+  }
+  throw Error(`${name} failed: ${errorMessage}`)
+}
+
+export async function retryHttpClientRequest<T>(
+  name: string,
+  method: () => Promise<IHttpClientResponse>,
+  customErrorMessages: Map<number, string> = new Map(),
+  maxAttempts = getRetryLimit()
+): Promise<IHttpClientResponse> {
+  return await retry(name, method, customErrorMessages, maxAttempts)
+}