From da1ec6e4855e54fe10d674c1152cb6b56f6cac80 Mon Sep 17 00:00:00 2001 From: Thiyagu K Date: Wed, 13 Aug 2025 07:34:22 +0000 Subject: [PATCH 1/2] fix-Set crc32c as the default option --- src/transfer-manager.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index dd4e41eeb..a0b5d5b08 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -127,7 +127,7 @@ export interface UploadFileInChunksOptions { uploadId?: string; autoAbortFailure?: boolean; partsMap?: Map; - validation?: 'md5' | false; + validation?: 'md5' | 'crc32c' | false; headers?: {[key: string]: string}; } @@ -140,7 +140,7 @@ export interface MultiPartUploadHelper { uploadPart( partNumber: number, chunk: Buffer, - validation?: 'md5' | false + validation?: 'md5' | 'crc32c' | false ): Promise; completeUpload(): Promise; abortUpload(): Promise; @@ -289,7 +289,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { async uploadPart( partNumber: number, chunk: Buffer, - validation?: 'md5' | false + validation?: 'md5' | 'crc32c' | false ): Promise { const url = `${this.baseUrl}?partNumber=${partNumber}&uploadId=${this.uploadId}`; let headers: Headers = this.#setGoogApiClientHeaders(); @@ -299,6 +299,10 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { headers = { 'Content-MD5': hash, }; + } else if (validation === 'crc32c') { + const crc = new CRC32C(); + crc.update(chunk); + headers['x-goog-hash'] = `crc32c=${crc.toString()}`; } return AsyncRetry(async bail => { @@ -806,6 +810,8 @@ export class TransferManager { ); let partNumber = 1; let promises: Promise[] = []; + const validation = + options.validation === undefined ? 'crc32c' : options.validation; try { if (options.uploadId === undefined) { await mpuHelper.initiateUpload(options.headers); @@ -823,9 +829,7 @@ export class TransferManager { promises = []; } promises.push( - limit(() => - mpuHelper.uploadPart(partNumber++, curChunk, options.validation) - ) + limit(() => mpuHelper.uploadPart(partNumber++, curChunk, validation)) ); } await Promise.all(promises); From 7bb00e5f7de10b57bf88a68ca339daa8d9103d98 Mon Sep 17 00:00:00 2001 From: Thiyagu K Date: Mon, 18 Aug 2025 10:04:52 +0000 Subject: [PATCH 2/2] added test cases --- test/transfer-manager.ts | 57 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/test/transfer-manager.ts b/test/transfer-manager.ts index 2582782fa..de0284ceb 100644 --- a/test/transfer-manager.ts +++ b/test/transfer-manager.ts @@ -730,5 +730,62 @@ describe('Transfer Manager', () => { assert(called); }); + + it('should use CRC32C validation when specified', async () => { + mockGeneratorFunction = (bucket, fileName, uploadId, partsMap) => { + fakeHelper = sandbox.createStubInstance(FakeXMLHelper); + fakeHelper.uploadId = uploadId || ''; + fakeHelper.partsMap = partsMap || new Map(); + fakeHelper.initiateUpload.resolves(); + fakeHelper.uploadPart.callsFake((partNumber, chunk, validation) => { + assert.strictEqual(validation, 'crc32c'); + + return Promise.resolve(); + }); + fakeHelper.completeUpload.resolves(); + fakeHelper.abortUpload.resolves(); + return fakeHelper; + }; + + await transferManager.uploadFileInChunks( + filePath, + {validation: 'crc32c'}, + mockGeneratorFunction + ); + + assert.strictEqual(fakeHelper.uploadPart.calledOnce, true); + }); + + it('should apply crc32c validation by default', async () => { + let assertionMade = false; + + mockGeneratorFunction = (bucket, fileName, uploadId, partsMap) => { + fakeHelper = sandbox.createStubInstance(FakeXMLHelper); + fakeHelper.uploadId = uploadId || ''; + fakeHelper.partsMap = partsMap || new Map(); + fakeHelper.initiateUpload.resolves(); + + fakeHelper.uploadPart.callsFake((partNumber, chunk, validation) => { + // Confirm the validation is set to 'crc32c' by default. + assert.strictEqual(validation, 'crc32c'); + assertionMade = true; + return Promise.resolve(); + }); + + fakeHelper.completeUpload.resolves(); + fakeHelper.abortUpload.resolves(); + return fakeHelper; + }; + + // Call the function without specifying any validation option. + await transferManager.uploadFileInChunks( + filePath, + {}, + mockGeneratorFunction + ); + + assert.strictEqual(fakeHelper.uploadPart.calledOnce, true); + assert.strictEqual(assertionMade, true); + }); }); });