From 72c0042dbe8859593ff19be253127b6f65e0c2cf Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Sun, 2 May 2021 02:49:33 +0000 Subject: [PATCH 1/9] add downloadAndDecryptAttachments for downloading and decrypting attachments --- README.md | 1 + package-lock.json | 33 ++++++++++++++++++++++++++++++++ package.json | 3 +++ spec/crypto.spec.ts | 46 +++++++++++++++++++++++++++++++++++++++++++++ src/crypto.ts | 42 +++++++++++++++++++++++++++++++++++++++++ src/types.ts | 13 +++++++++++++ 6 files changed, 138 insertions(+) diff --git a/README.md b/README.md index abab919..2915d3b 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ The underlying cryptosystem is `x25519-xsalsa20-poly1305` which is implemented b | submissionId | string | Unique response identifier, displayed as 'Response ID' to form respondents | | encryptedContent | string | The encrypted submission in base64. | | created | string | Creation timestamp. | +| attachmentDownloadUrls | Record | (Optional) Records containing field IDs and URLs where encrypted uploaded attachments can be downloaded. | ### Format of Decrypted Submissions diff --git a/package-lock.json b/package-lock.json index b65225f..7fe30b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1436,6 +1436,11 @@ "type-detect": "4.0.8" } }, + "@stablelib/base64": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.0.tgz", + "integrity": "sha512-s/wTc/3+vYSalh4gfayJrupzhT7SDBqNtiYOeEMlkSDqL/8cExh5FAeTzLpmYq+7BLLv36EjBL5xrb0bUHWJWQ==" + }, "@types/babel__core": { "version": "7.1.7", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.7.tgz", @@ -1759,6 +1764,14 @@ "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==", "dev": true }, + "axios": { + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", + "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "requires": { + "follow-redirects": "^1.10.0" + } + }, "babel-jest": { "version": "25.3.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-25.3.0.tgz", @@ -2765,6 +2778,11 @@ "locate-path": "^3.0.0" } }, + "follow-redirects": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.0.tgz", + "integrity": "sha512-0vRwd7RKQBTt+mgu87mtYeofLFZpTas2S9zY+jIeuLJMNvudIgF52nr19q40HOwH5RrhWIPuj9puybzSJiRrVg==" + }, "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", @@ -4382,6 +4400,15 @@ "@jest/types": "^25.3.0" } }, + "jest-mock-axios": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/jest-mock-axios/-/jest-mock-axios-4.4.0.tgz", + "integrity": "sha512-MF5MbjIZcv2KCtO6oH/Fmy1sML1LxQoaGyIPRGArDdd9pcsfjWoCmFFUD12GgOTeJw8ChjuVYHN0s8QnhGriQA==", + "dev": true, + "requires": { + "synchronous-promise": "^2.0.15" + } + }, "jest-pnp-resolver": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.1.tgz", @@ -6111,6 +6138,12 @@ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "dev": true }, + "synchronous-promise": { + "version": "2.0.15", + "resolved": "https://registry.npmjs.org/synchronous-promise/-/synchronous-promise-2.0.15.tgz", + "integrity": "sha512-k8uzYIkIVwmT+TcglpdN50pS2y1BDcUnBPK9iJeGu0Pl1lOI8pD6wtzgw91Pjpe+RxtTncw32tLxs/R0yNL2Mg==", + "dev": true + }, "terminal-link": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", diff --git a/package.json b/package.json index 8746c0e..bcb3191 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,8 @@ "author": "Open Government Products (FormSG)", "license": "MIT", "dependencies": { + "@stablelib/base64": "^1.0.0", + "axios": "^0.21.1", "tweetnacl": "^1.0.3", "tweetnacl-util": "^0.15.1" }, @@ -34,6 +36,7 @@ "@types/node": "^13.11.1", "coveralls": "^3.1.0", "jest": "^25.3.0", + "jest-mock-axios": "^4.4.0", "ts-jest": "^25.3.1", "typescript": "^3.8.3" } diff --git a/spec/crypto.spec.ts b/spec/crypto.spec.ts index 3e85857..ea0d186 100644 --- a/spec/crypto.spec.ts +++ b/spec/crypto.spec.ts @@ -1,6 +1,11 @@ +import mockAxios from 'jest-mock-axios' import Crypto from '../src/crypto' import { SIGNING_KEYS } from '../src/resource/signing-keys' +import { + encodeBase64, +} from 'tweetnacl-util' + import { plaintext, ciphertext, @@ -16,7 +21,11 @@ const encryptionPublicKey = SIGNING_KEYS.test.publicKey const signingSecretKey = SIGNING_KEYS.test.secretKey const testFileBuffer = new Uint8Array(Buffer.from('./resources/ogp.svg')) +jest.mock('axios', () => mockAxios) + describe('Crypto', function () { + afterEach(() => mockAxios.reset()) + const crypto = new Crypto({ signingPublicKey: encryptionPublicKey }) const mockVerifiedContent = { @@ -231,4 +240,41 @@ describe('Crypto', function () { }) expect(decryptResult).toBeNull() }) + + it('should be able to download and decrypt an attachment successfully', async () => { + // Arrange + const { publicKey, secretKey } = crypto.generate() + + let attachmentPlaintext = plaintext + attachmentPlaintext.push({ + _id: '6e771c946b3c5100240368e5', + question: 'Random file', + fieldType: 'attachment', + answer: 'my-random-file.txt', + }) + + // Encrypt content that is not signed + const ciphertext = crypto.encrypt(attachmentPlaintext, publicKey) + + // Encrypt file + const encryptedFile = await crypto.encryptFile(testFileBuffer, publicKey) + const uploadedFile = { + submissionPublicKey: encryptedFile.submissionPublicKey, + nonce: encryptedFile.nonce, + binary: encodeBase64(encryptedFile.binary) + } + + // Act + const decryptedFilesPromise = crypto.downloadAndDecryptAttachments(secretKey, { + encryptedContent: ciphertext, + attachmentDownloadUrls: { '6e771c946b3c5100240368e5': 'https://some.s3.url/some/encrypted/file' }, + version: INTERNAL_TEST_VERSION, + }) + mockAxios.mockResponse({ data: { encryptedFile: uploadedFile }}) + const decryptedFiles = await(decryptedFilesPromise) + + // Assert + expect(mockAxios.get).toHaveBeenCalledWith('https://some.s3.url/some/encrypted/file', { responseType: 'json' }) + expect(decryptedFiles).toHaveProperty('6e771c946b3c5100240368e5', { filename: 'my-random-file.txt', content: testFileBuffer }) + }) }) diff --git a/src/crypto.ts b/src/crypto.ts index 54cac76..36bf6c8 100644 --- a/src/crypto.ts +++ b/src/crypto.ts @@ -1,8 +1,11 @@ +import axios from 'axios' import nacl from 'tweetnacl' import { DecryptParams, + DecryptedAttachments, DecryptedContent, + EncryptedAttachmentRecords, EncryptedContent, EncryptedFileContent, FormField, @@ -15,6 +18,8 @@ import { decodeUTF8, } from 'tweetnacl-util' +import { decode as decodeBase64ToUint8Array } from '@stablelib/base64' + import { determineIsFormFields } from './util/validate' import { MissingPublicKeyError } from './errors' import { @@ -195,4 +200,41 @@ export default class Crypto { decodeBase64(formSecretKey) ) } + + /** + * Download and decrypt the given attachments. + * @param formSecretKey Secret key as a base-64 string + * @param decryptParams The params containing encrypted content and information. + * @returns The decrypted attachments if successful. Else, null will be returned. + * @throws {MissingPublicKeyError} if a public key is not provided when instantiating this class and is needed for verifying signed content. + */ + downloadAndDecryptAttachments = async(formSecretKey: string, decryptParams: DecryptParams): Promise => { + let decryptedRecords: DecryptedAttachments = {} + + const attachmentRecords: EncryptedAttachmentRecords = decryptParams.attachmentDownloadUrls ?? {} + const decryptedContent = this.decrypt(formSecretKey, decryptParams) + if (decryptedContent === null) return null + + for (let fieldId in attachmentRecords) { + const downloadResponse = await axios.get(attachmentRecords[fieldId], { responseType: 'json' }) + if (downloadResponse.status !== 200) return null + + let data = downloadResponse.data + data.encryptedFile.binary = decodeBase64ToUint8Array(data.encryptedFile.binary) + const decryptedFile = await this.decryptFile(formSecretKey, data.encryptedFile) + + let filename = 'Unknown File.data' + for (const i in decryptedContent.responses) { + const response = decryptedContent.responses[i] + if (response._id === fieldId && response.answer) { + filename = response.answer + break + } + } + + decryptedRecords[fieldId] = { filename, content: decryptedFile } + } + + return decryptedRecords + } } diff --git a/src/types.ts b/src/types.ts index 09adafe..db8e218 100644 --- a/src/types.ts +++ b/src/types.ts @@ -45,10 +45,15 @@ export type FormField = { // ;: export type EncryptedContent = string +// Records containing a map of field IDs to URLs where encrypted +// attachments can be downloaded. +export type EncryptedAttachmentRecords = Record + export interface DecryptParams { encryptedContent: EncryptedContent version: number verifiedContent?: EncryptedContent + attachmentDownloadUrls?: EncryptedAttachmentRecords } export type DecryptedContent = { @@ -56,6 +61,14 @@ export type DecryptedContent = { verified?: Record } +export type DecryptedFile = { + filename: string + content: Uint8Array | null +} + +// Records containing a map of field IDs to DecryptedFiles. +export type DecryptedAttachments = Record + export type EncryptedFileContent = { submissionPublicKey: string nonce: string From 0eff7cb15dd1610c4a29c6efe4b3a79a539e2ce4 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Sun, 2 May 2021 03:40:23 +0000 Subject: [PATCH 2/9] add more tests to improve coverage --- spec/crypto.spec.ts | 74 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/spec/crypto.spec.ts b/spec/crypto.spec.ts index ea0d186..d661739 100644 --- a/spec/crypto.spec.ts +++ b/spec/crypto.spec.ts @@ -271,10 +271,82 @@ describe('Crypto', function () { version: INTERNAL_TEST_VERSION, }) mockAxios.mockResponse({ data: { encryptedFile: uploadedFile }}) - const decryptedFiles = await(decryptedFilesPromise) + const decryptedFiles = await decryptedFilesPromise // Assert expect(mockAxios.get).toHaveBeenCalledWith('https://some.s3.url/some/encrypted/file', { responseType: 'json' }) expect(decryptedFiles).toHaveProperty('6e771c946b3c5100240368e5', { filename: 'my-random-file.txt', content: testFileBuffer }) }) + + it('should be able to handle fields without attachmentDownloadUrls', async () => { + // Arrange + const { publicKey, secretKey } = crypto.generate() + + // Encrypt content that is not signed + const ciphertext = crypto.encrypt(plaintext, publicKey) + + // Act + const decryptedFiles = await crypto.downloadAndDecryptAttachments(secretKey, { + encryptedContent: ciphertext, + version: INTERNAL_TEST_VERSION, + }) + + // Assert + expect(decryptedFiles).toEqual({}) + }) + + it('should be able to handle corrupted encrypted content', async () => { + // Arrange + const { secretKey } = crypto.generate() + + // Act + const decryptedFiles = await crypto.downloadAndDecryptAttachments(secretKey, { + encryptedContent: 'bad encrypted content', + version: INTERNAL_TEST_VERSION, + }) + + // Assert + expect(decryptedFiles).toBe(null) + }) + + it('should be able to handle axios errors', async () => { + // Arrange + const { publicKey, secretKey } = crypto.generate() + + let attachmentPlaintext = plaintext + attachmentPlaintext.push({ + _id: '6e771c946b3c5100240368e5', + question: 'Random file', + fieldType: 'attachment', + answer: 'my-random-file.txt', + }) + + // Encrypt content that is not signed + const ciphertext = crypto.encrypt(attachmentPlaintext, publicKey) + + // Encrypt file + const encryptedFile = await crypto.encryptFile(testFileBuffer, publicKey) + const uploadedFile = { + submissionPublicKey: encryptedFile.submissionPublicKey, + nonce: encryptedFile.nonce, + binary: encodeBase64(encryptedFile.binary) + } + + // Act + const decryptedFilesPromise = crypto.downloadAndDecryptAttachments(secretKey, { + encryptedContent: ciphertext, + attachmentDownloadUrls: { '6e771c946b3c5100240368e5': 'https://some.s3.url/some/encrypted/file' }, + version: INTERNAL_TEST_VERSION, + }) + mockAxios.mockResponse({ + data: {}, + status: 404, + statusText: 'Not Found', + }) + const decryptedFiles = await decryptedFilesPromise + + // Assert + expect(mockAxios.get).toHaveBeenCalledWith('https://some.s3.url/some/encrypted/file', { responseType: 'json' }) + expect(decryptedFiles).toBe(null) + }) }) From a118c897577e5655c4b3102a1f30afe1b2b4e733 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 4 May 2021 22:26:32 -0700 Subject: [PATCH 3/9] Update src/crypto.ts Co-authored-by: Antariksh Mahajan --- src/crypto.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/crypto.ts b/src/crypto.ts index 36bf6c8..657a389 100644 --- a/src/crypto.ts +++ b/src/crypto.ts @@ -208,7 +208,10 @@ export default class Crypto { * @returns The decrypted attachments if successful. Else, null will be returned. * @throws {MissingPublicKeyError} if a public key is not provided when instantiating this class and is needed for verifying signed content. */ - downloadAndDecryptAttachments = async(formSecretKey: string, decryptParams: DecryptParams): Promise => { + downloadAndDecryptAttachments = async ( + formSecretKey: string, + decryptParams: DecryptParams + ): Promise => { let decryptedRecords: DecryptedAttachments = {} const attachmentRecords: EncryptedAttachmentRecords = decryptParams.attachmentDownloadUrls ?? {} From 6c81aa66d45b26996771ea50a1f62618c31b00a4 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 5 May 2021 07:08:13 +0000 Subject: [PATCH 4/9] update code per comments --- README.md | 16 +++++++++ package-lock.json | 5 --- package.json | 1 - spec/crypto.spec.ts | 81 +++++++++++++++++++++++++++++++++++++++------ src/crypto.ts | 53 ++++++++++++++++++----------- src/types.ts | 15 ++++++++- 6 files changed, 133 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 2915d3b..ee23fcb 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,22 @@ If the decrypted content is the correct shape, then: verified content. **If the verification fails, `null` is returned, even if `decryptParams.encryptedContent` was successfully decrypted.** +### Processing Attachments + +`formsg.crypto.decryptWithAttachments(formSecretKey: string, decryptParams: DecryptParams) behaves similarly except it will return a `Promise`. + +`DecryptedContentAndAttachments` is an object containing two fields: + - `content`: the standard form decrypted responses (same as the return type of `formsg.crypto.decrypt`) + - `attachments`: A `Record` containing a map of field ids of the attachment fields to a object containing the original user supplied filename and a `Uint8Array` containing the contents of the uploaded file. + +If the contents of any file fails to decrypt or there is a mismatch between the attachments and submission (e.g. the submission doesn't contain the original file name), then `null` will be returned. + +Attachments are downloaded using S3 pre-signed URLs, with a expiry time of *one hour*. You must call `decryptWithAttachments` within this time window, or else the URL to the encrypted files will become invalid. + +Attachments are end-to-end encrypted in the same way as normal form submissions, so any eavesdropper will not be able to view form attachments without your secret key. + +*Warning:* We do not have the ability to scan any attachments for malicious content (e.g. spyware or viruses), so careful handling is neeeded. + ## Verifying Signatures Manually You can use the following information to create a custom solution, although we recommend using this SDK. diff --git a/package-lock.json b/package-lock.json index 7fe30b0..f1ee8b3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1436,11 +1436,6 @@ "type-detect": "4.0.8" } }, - "@stablelib/base64": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.0.tgz", - "integrity": "sha512-s/wTc/3+vYSalh4gfayJrupzhT7SDBqNtiYOeEMlkSDqL/8cExh5FAeTzLpmYq+7BLLv36EjBL5xrb0bUHWJWQ==" - }, "@types/babel__core": { "version": "7.1.7", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.7.tgz", diff --git a/package.json b/package.json index bcb3191..7bd16a3 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,6 @@ "author": "Open Government Products (FormSG)", "license": "MIT", "dependencies": { - "@stablelib/base64": "^1.0.0", "axios": "^0.21.1", "tweetnacl": "^1.0.3", "tweetnacl-util": "^0.15.1" diff --git a/spec/crypto.spec.ts b/spec/crypto.spec.ts index d661739..d97f6b6 100644 --- a/spec/crypto.spec.ts +++ b/spec/crypto.spec.ts @@ -245,7 +245,7 @@ describe('Crypto', function () { // Arrange const { publicKey, secretKey } = crypto.generate() - let attachmentPlaintext = plaintext + let attachmentPlaintext = plaintext.slice(0) attachmentPlaintext.push({ _id: '6e771c946b3c5100240368e5', question: 'Random file', @@ -265,13 +265,14 @@ describe('Crypto', function () { } // Act - const decryptedFilesPromise = crypto.downloadAndDecryptAttachments(secretKey, { + const decryptedFilesPromise = crypto.decryptWithAttachments(secretKey, { encryptedContent: ciphertext, attachmentDownloadUrls: { '6e771c946b3c5100240368e5': 'https://some.s3.url/some/encrypted/file' }, version: INTERNAL_TEST_VERSION, }) mockAxios.mockResponse({ data: { encryptedFile: uploadedFile }}) - const decryptedFiles = await decryptedFilesPromise + const decryptedContentWithAttachments = await decryptedFilesPromise + const decryptedFiles = decryptedContentWithAttachments!.attachments // Assert expect(mockAxios.get).toHaveBeenCalledWith('https://some.s3.url/some/encrypted/file', { responseType: 'json' }) @@ -286,10 +287,11 @@ describe('Crypto', function () { const ciphertext = crypto.encrypt(plaintext, publicKey) // Act - const decryptedFiles = await crypto.downloadAndDecryptAttachments(secretKey, { + const decryptedContentWithAttachments = await crypto.decryptWithAttachments(secretKey, { encryptedContent: ciphertext, version: INTERNAL_TEST_VERSION, }) + const decryptedFiles = decryptedContentWithAttachments!.attachments // Assert expect(decryptedFiles).toEqual({}) @@ -300,20 +302,20 @@ describe('Crypto', function () { const { secretKey } = crypto.generate() // Act - const decryptedFiles = await crypto.downloadAndDecryptAttachments(secretKey, { + const decryptedContents = await crypto.decryptWithAttachments(secretKey, { encryptedContent: 'bad encrypted content', version: INTERNAL_TEST_VERSION, }) // Assert - expect(decryptedFiles).toBe(null) + expect(decryptedContents).toBe(null) }) - it('should be able to handle axios errors', async () => { + it('should be able to handle corrupted download', async () => { // Arrange const { publicKey, secretKey } = crypto.generate() - let attachmentPlaintext = plaintext + let attachmentPlaintext = plaintext.slice(0) attachmentPlaintext.push({ _id: '6e771c946b3c5100240368e5', question: 'Random file', @@ -324,6 +326,35 @@ describe('Crypto', function () { // Encrypt content that is not signed const ciphertext = crypto.encrypt(attachmentPlaintext, publicKey) + // Encrypt file + const encryptedFile = await crypto.encryptFile(testFileBuffer, publicKey) + const uploadedFile = { + submissionPublicKey: encryptedFile.submissionPublicKey, + nonce: encryptedFile.nonce, + binary: 'YmFkZW5jcnlwdGVkY29udGVudHM=', // invalid data + } + + // Act + const decryptedFilesPromise = crypto.decryptWithAttachments(secretKey, { + encryptedContent: ciphertext, + attachmentDownloadUrls: { '6e771c946b3c5100240368e5': 'https://some.s3.url/some/encrypted/file' }, + version: INTERNAL_TEST_VERSION, + }) + mockAxios.mockResponse({ data: { encryptedFile: uploadedFile }}) + const decryptedContents = await decryptedFilesPromise + + // Assert + expect(decryptedContents).toBe(null) + }) + + it('should be able to handle decrypted submission without corresponding attachment field', async () => { + // Arrange + const { publicKey, secretKey } = crypto.generate() + + // Encrypt content that is not signed + // Note that plaintext doesn't have any attachment fields + const ciphertext = crypto.encrypt(plaintext, publicKey) + // Encrypt file const encryptedFile = await crypto.encryptFile(testFileBuffer, publicKey) const uploadedFile = { @@ -333,7 +364,35 @@ describe('Crypto', function () { } // Act - const decryptedFilesPromise = crypto.downloadAndDecryptAttachments(secretKey, { + const decryptedFilesPromise = crypto.decryptWithAttachments(secretKey, { + encryptedContent: ciphertext, + attachmentDownloadUrls: { '6e771c946b3c5100240368e5': 'https://some.s3.url/some/encrypted/file' }, + version: INTERNAL_TEST_VERSION, + }) + mockAxios.mockResponse({ data: { encryptedFile: uploadedFile }}) + const decryptedContents = await decryptedFilesPromise + + // Assert + expect(decryptedContents).toBe(null) + }) + + it('should be able to handle axios errors', async () => { + // Arrange + const { publicKey, secretKey } = crypto.generate() + + let attachmentPlaintext = plaintext.slice(0) + attachmentPlaintext.push({ + _id: '6e771c946b3c5100240368e5', + question: 'Random file', + fieldType: 'attachment', + answer: 'my-random-file.txt', + }) + + // Encrypt content that is not signed + const ciphertext = crypto.encrypt(attachmentPlaintext, publicKey) + + // Act + const decryptedFilesPromise = crypto.decryptWithAttachments(secretKey, { encryptedContent: ciphertext, attachmentDownloadUrls: { '6e771c946b3c5100240368e5': 'https://some.s3.url/some/encrypted/file' }, version: INTERNAL_TEST_VERSION, @@ -343,10 +402,10 @@ describe('Crypto', function () { status: 404, statusText: 'Not Found', }) - const decryptedFiles = await decryptedFilesPromise + const decryptedContents = await decryptedFilesPromise // Assert expect(mockAxios.get).toHaveBeenCalledWith('https://some.s3.url/some/encrypted/file', { responseType: 'json' }) - expect(decryptedFiles).toBe(null) + expect(decryptedContents).toBe(null) }) }) diff --git a/src/crypto.ts b/src/crypto.ts index 657a389..888df4a 100644 --- a/src/crypto.ts +++ b/src/crypto.ts @@ -5,6 +5,8 @@ import { DecryptParams, DecryptedAttachments, DecryptedContent, + DecryptedContentAndAttachments, + EncryptedAttachmentContent, EncryptedAttachmentRecords, EncryptedContent, EncryptedFileContent, @@ -18,8 +20,6 @@ import { decodeUTF8, } from 'tweetnacl-util' -import { decode as decodeBase64ToUint8Array } from '@stablelib/base64' - import { determineIsFormFields } from './util/validate' import { MissingPublicKeyError } from './errors' import { @@ -202,42 +202,55 @@ export default class Crypto { } /** - * Download and decrypt the given attachments. + * Decrypts an encrypted submission, and also download and decrypt any attachments alongside it. * @param formSecretKey Secret key as a base-64 string * @param decryptParams The params containing encrypted content and information. - * @returns The decrypted attachments if successful. Else, null will be returned. + * @returns A promise of the decrypted submission, including attachments (if any). Or else returns null if a decryption error decrypting any part of the submission. * @throws {MissingPublicKeyError} if a public key is not provided when instantiating this class and is needed for verifying signed content. */ - downloadAndDecryptAttachments = async ( + decryptWithAttachments = async ( formSecretKey: string, decryptParams: DecryptParams - ): Promise => { - let decryptedRecords: DecryptedAttachments = {} + ): Promise => { + const decryptedRecords: DecryptedAttachments = {} + const filenames: Record = {} const attachmentRecords: EncryptedAttachmentRecords = decryptParams.attachmentDownloadUrls ?? {} const decryptedContent = this.decrypt(formSecretKey, decryptParams) if (decryptedContent === null) return null + // Retrieve all original filenames for attachments for easy lookup + for (const i in decryptedContent.responses) { + const response = decryptedContent.responses[i] + if (response.fieldType === 'attachment' && response.answer) { + filenames[response._id] = response.answer + } + } + for (let fieldId in attachmentRecords) { const downloadResponse = await axios.get(attachmentRecords[fieldId], { responseType: 'json' }) if (downloadResponse.status !== 200) return null - let data = downloadResponse.data - data.encryptedFile.binary = decodeBase64ToUint8Array(data.encryptedFile.binary) - const decryptedFile = await this.decryptFile(formSecretKey, data.encryptedFile) - - let filename = 'Unknown File.data' - for (const i in decryptedContent.responses) { - const response = decryptedContent.responses[i] - if (response._id === fieldId && response.answer) { - filename = response.answer - break - } + const encryptedAttachment: EncryptedAttachmentContent = downloadResponse.data + const encryptedFile: EncryptedFileContent = { + submissionPublicKey: encryptedAttachment.encryptedFile.submissionPublicKey, + nonce: encryptedAttachment.encryptedFile.nonce, + binary: decodeBase64(encryptedAttachment.encryptedFile.binary), } + const decryptedFile = await this.decryptFile(formSecretKey, encryptedFile) + + // Decryption for a file failed + if (decryptedFile === null) return null + + // Original name for the file is not found + if (filenames[fieldId] === undefined) return null - decryptedRecords[fieldId] = { filename, content: decryptedFile } + decryptedRecords[fieldId] = { filename: filenames[fieldId], content: decryptedFile } } - return decryptedRecords + return { + content: decryptedContent, + attachments: decryptedRecords + } } } diff --git a/src/types.ts b/src/types.ts index db8e218..96b7e68 100644 --- a/src/types.ts +++ b/src/types.ts @@ -63,18 +63,31 @@ export type DecryptedContent = { export type DecryptedFile = { filename: string - content: Uint8Array | null + content: Uint8Array } // Records containing a map of field IDs to DecryptedFiles. export type DecryptedAttachments = Record +export type DecryptedContentAndAttachments = { + content: DecryptedContent + attachments: DecryptedAttachments +} + export type EncryptedFileContent = { submissionPublicKey: string nonce: string binary: Uint8Array } +export type EncryptedAttachmentContent = { + encryptedFile: { + submissionPublicKey: string + nonce: string + binary: string + } +} + // A base-64 encoded cryptographic keypair suitable for curve25519. export type Keypair = { publicKey: string From 497266310712222c3eceeeb5a2a4a900caa82761 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 5 May 2021 07:09:26 +0000 Subject: [PATCH 5/9] add a backtick to readme.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ee23fcb..f494b7b 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ If the decrypted content is the correct shape, then: ### Processing Attachments -`formsg.crypto.decryptWithAttachments(formSecretKey: string, decryptParams: DecryptParams) behaves similarly except it will return a `Promise`. +`formsg.crypto.decryptWithAttachments(formSecretKey: string, decryptParams: DecryptParams)` behaves similarly except it will return a `Promise`. `DecryptedContentAndAttachments` is an object containing two fields: - `content`: the standard form decrypted responses (same as the return type of `formsg.crypto.decrypt`) From cbbe9ac8923fd16de6940d2fa19059d372bf87af Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 7 May 2021 06:57:06 +0000 Subject: [PATCH 6/9] address comments by @seaerchin and @mantariksh --- README.md | 43 +++++++++++++++++++++++++++++++++++++++++++ spec/crypto.spec.ts | 1 - src/crypto.ts | 42 +++++++++++++++++++++++++----------------- 3 files changed, 68 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index f494b7b..fe1ddb7 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,49 @@ app.post( } ) +// Example for submissions with attachments +app.post( + '/submissions-attachment', + // Endpoint authentication by verifying signatures + function (req, res, next) { + try { + formsg.webhooks.authenticate(req.get('X-FormSG-Signature'), POST_URI) + // Continue processing the POST body + return next() + } catch (e) { + return res.status(401).send({ message: 'Unauthorized' }) + } + }, + // Parse JSON from raw request body + express.json(), + // Decrypt the submission and attachments + async function (req, res, next) { + // `req.body.data` is an object fulfilling the DecryptParams interface. + // interface DecryptParams { + // encryptedContent: EncryptedContent + // version: number + // verifiedContent?: EncryptedContent + // } + /** @type {{content: DecryptedContent, attachments: DecryptedAttachments}} */ + const submission = formsg.crypto.decryptWithAttachments( + formSecretKey, + // If `verifiedContent` is provided in `req.body.data`, the return object + // will include a verified key. + req.body.data + ) + + // If the decryption failed, submission will be `null`. + if (submission) { + // Continue processing the submission + + // processSubmission(submission.content) + // processAttachments(submission.attachments) + } else { + // Could not decrypt the submission + } + } +) + app.listen(8080, () => console.log('Running on port 8080')) ``` diff --git a/spec/crypto.spec.ts b/spec/crypto.spec.ts index d97f6b6..7b325e7 100644 --- a/spec/crypto.spec.ts +++ b/spec/crypto.spec.ts @@ -369,7 +369,6 @@ describe('Crypto', function () { attachmentDownloadUrls: { '6e771c946b3c5100240368e5': 'https://some.s3.url/some/encrypted/file' }, version: INTERNAL_TEST_VERSION, }) - mockAxios.mockResponse({ data: { encryptedFile: uploadedFile }}) const decryptedContents = await decryptedFilesPromise // Assert diff --git a/src/crypto.ts b/src/crypto.ts index 888df4a..bb19975 100644 --- a/src/crypto.ts +++ b/src/crypto.ts @@ -220,32 +220,40 @@ export default class Crypto { if (decryptedContent === null) return null // Retrieve all original filenames for attachments for easy lookup - for (const i in decryptedContent.responses) { - const response = decryptedContent.responses[i] + decryptedContent.responses.forEach((response) => { if (response.fieldType === 'attachment' && response.answer) { filenames[response._id] = response.answer } - } + }) + const downloadPromises : Array> = [] for (let fieldId in attachmentRecords) { - const downloadResponse = await axios.get(attachmentRecords[fieldId], { responseType: 'json' }) - if (downloadResponse.status !== 200) return null + // Original name for the file is not found + if (filenames[fieldId] === undefined) return null - const encryptedAttachment: EncryptedAttachmentContent = downloadResponse.data - const encryptedFile: EncryptedFileContent = { - submissionPublicKey: encryptedAttachment.encryptedFile.submissionPublicKey, - nonce: encryptedAttachment.encryptedFile.nonce, - binary: decodeBase64(encryptedAttachment.encryptedFile.binary), - } - const decryptedFile = await this.decryptFile(formSecretKey, encryptedFile) + downloadPromises.push( + axios.get(attachmentRecords[fieldId], { responseType: 'json' }) + .then((downloadResponse) => { + if (downloadResponse.status !== 200) throw new Error("Download failed") - // Decryption for a file failed - if (decryptedFile === null) return null + const encryptedAttachment: EncryptedAttachmentContent = downloadResponse.data + const encryptedFile: EncryptedFileContent = { + submissionPublicKey: encryptedAttachment.encryptedFile.submissionPublicKey, + nonce: encryptedAttachment.encryptedFile.nonce, + binary: decodeBase64(encryptedAttachment.encryptedFile.binary), + } - // Original name for the file is not found - if (filenames[fieldId] === undefined) return null + return this.decryptFile(formSecretKey, encryptedFile) + }).then((decryptedFile) => { + if (decryptedFile === null) throw new Error("Attachment decryption failed") + decryptedRecords[fieldId] = { filename: filenames[fieldId], content: decryptedFile } + })) + } - decryptedRecords[fieldId] = { filename: filenames[fieldId], content: decryptedFile } + try { + await Promise.all(downloadPromises) + } catch (e) { + return null } return { From 46d359365a85438a93cff9e1e56a2914a6664e3d Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 7 May 2021 06:58:29 +0000 Subject: [PATCH 7/9] update types in README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fe1ddb7..5d2493e 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,7 @@ app.post( // encryptedContent: EncryptedContent // version: number // verifiedContent?: EncryptedContent + // attachmentDownloadUrls?: Record // } /** @type {{content: DecryptedContent, attachments: DecryptedAttachments}} */ const submission = formsg.crypto.decryptWithAttachments( @@ -117,7 +118,7 @@ app.post( req.body.data ) - // If the decryption failed, submission will be `null`. + // If the decryption failed at any point, submission will be `null`. if (submission) { // Continue processing the submission From 0fe7082673fb241727dcc4caa12b6de2fa55d8c1 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 10 May 2021 19:17:39 -0700 Subject: [PATCH 8/9] remove extraneous http status check --- src/crypto.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/crypto.ts b/src/crypto.ts index bb19975..af774b1 100644 --- a/src/crypto.ts +++ b/src/crypto.ts @@ -234,8 +234,6 @@ export default class Crypto { downloadPromises.push( axios.get(attachmentRecords[fieldId], { responseType: 'json' }) .then((downloadResponse) => { - if (downloadResponse.status !== 200) throw new Error("Download failed") - const encryptedAttachment: EncryptedAttachmentContent = downloadResponse.data const encryptedFile: EncryptedFileContent = { submissionPublicKey: encryptedAttachment.encryptedFile.submissionPublicKey, From 512ad0a6fb578d9eb6a61bf44cfa9b67e302b4db Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 10 May 2021 22:53:32 -0700 Subject: [PATCH 9/9] remove type annotations in readme.md --- README.md | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/README.md b/README.md index 5d2493e..f9c206d 100644 --- a/README.md +++ b/README.md @@ -63,13 +63,6 @@ app.post( express.json(), // Decrypt the submission function (req, res, next) { - // `req.body.data` is an object fulfilling the DecryptParams interface. - // interface DecryptParams { - // encryptedContent: EncryptedContent - // version: number - // verifiedContent?: EncryptedContent - // } - /** @type {{responses: FormField[], verified?: Record}} */ const submission = formsg.crypto.decrypt( formSecretKey, // If `verifiedContent` is provided in `req.body.data`, the return object @@ -103,14 +96,6 @@ app.post( express.json(), // Decrypt the submission and attachments async function (req, res, next) { - // `req.body.data` is an object fulfilling the DecryptParams interface. - // interface DecryptParams { - // encryptedContent: EncryptedContent - // version: number - // verifiedContent?: EncryptedContent - // attachmentDownloadUrls?: Record - // } - /** @type {{content: DecryptedContent, attachments: DecryptedAttachments}} */ const submission = formsg.crypto.decryptWithAttachments( formSecretKey, // If `verifiedContent` is provided in `req.body.data`, the return object