diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index dd4e41eeb..a0b5d5b08 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -127,7 +127,7 @@ export interface UploadFileInChunksOptions { uploadId?: string; autoAbortFailure?: boolean; partsMap?: Map; - validation?: 'md5' | false; + validation?: 'md5' | 'crc32c' | false; headers?: {[key: string]: string}; } @@ -140,7 +140,7 @@ export interface MultiPartUploadHelper { uploadPart( partNumber: number, chunk: Buffer, - validation?: 'md5' | false + validation?: 'md5' | 'crc32c' | false ): Promise; completeUpload(): Promise; abortUpload(): Promise; @@ -289,7 +289,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { async uploadPart( partNumber: number, chunk: Buffer, - validation?: 'md5' | false + validation?: 'md5' | 'crc32c' | false ): Promise { const url = `${this.baseUrl}?partNumber=${partNumber}&uploadId=${this.uploadId}`; let headers: Headers = this.#setGoogApiClientHeaders(); @@ -299,6 +299,10 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { headers = { 'Content-MD5': hash, }; + } else if (validation === 'crc32c') { + const crc = new CRC32C(); + crc.update(chunk); + headers['x-goog-hash'] = `crc32c=${crc.toString()}`; } return AsyncRetry(async bail => { @@ -806,6 +810,8 @@ export class TransferManager { ); let partNumber = 1; let promises: Promise[] = []; + const validation = + options.validation === undefined ? 'crc32c' : options.validation; try { if (options.uploadId === undefined) { await mpuHelper.initiateUpload(options.headers); @@ -823,9 +829,7 @@ export class TransferManager { promises = []; } promises.push( - limit(() => - mpuHelper.uploadPart(partNumber++, curChunk, options.validation) - ) + limit(() => mpuHelper.uploadPart(partNumber++, curChunk, validation)) ); } await Promise.all(promises); diff --git a/test/transfer-manager.ts b/test/transfer-manager.ts index 2582782fa..de0284ceb 100644 --- a/test/transfer-manager.ts +++ b/test/transfer-manager.ts @@ -730,5 +730,62 @@ describe('Transfer Manager', () => { assert(called); }); + + it('should use CRC32C validation when specified', async () => { + mockGeneratorFunction = (bucket, fileName, uploadId, partsMap) => { + fakeHelper = sandbox.createStubInstance(FakeXMLHelper); + fakeHelper.uploadId = uploadId || ''; + fakeHelper.partsMap = partsMap || new Map(); + fakeHelper.initiateUpload.resolves(); + fakeHelper.uploadPart.callsFake((partNumber, chunk, validation) => { + assert.strictEqual(validation, 'crc32c'); + + return Promise.resolve(); + }); + fakeHelper.completeUpload.resolves(); + fakeHelper.abortUpload.resolves(); + return fakeHelper; + }; + + await transferManager.uploadFileInChunks( + filePath, + {validation: 'crc32c'}, + mockGeneratorFunction + ); + + assert.strictEqual(fakeHelper.uploadPart.calledOnce, true); + }); + + it('should apply crc32c validation by default', async () => { + let assertionMade = false; + + mockGeneratorFunction = (bucket, fileName, uploadId, partsMap) => { + fakeHelper = sandbox.createStubInstance(FakeXMLHelper); + fakeHelper.uploadId = uploadId || ''; + fakeHelper.partsMap = partsMap || new Map(); + fakeHelper.initiateUpload.resolves(); + + fakeHelper.uploadPart.callsFake((partNumber, chunk, validation) => { + // Confirm the validation is set to 'crc32c' by default. + assert.strictEqual(validation, 'crc32c'); + assertionMade = true; + return Promise.resolve(); + }); + + fakeHelper.completeUpload.resolves(); + fakeHelper.abortUpload.resolves(); + return fakeHelper; + }; + + // Call the function without specifying any validation option. + await transferManager.uploadFileInChunks( + filePath, + {}, + mockGeneratorFunction + ); + + assert.strictEqual(fakeHelper.uploadPart.calledOnce, true); + assert.strictEqual(assertionMade, true); + }); }); });