From d708e7333e61e17207465308c49823398caa3bfb Mon Sep 17 00:00:00 2001 From: Mart Date: Sun, 23 Jul 2023 17:22:00 +0800 Subject: [PATCH 01/19] add Xss.stripEmojis() --- .../elements/compose-modules/compose-input-module.ts | 6 +++--- .../compose-modules/compose-recipients-module.ts | 10 +++++----- extension/js/common/platform/xss.ts | 9 +++++++++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-input-module.ts b/extension/chrome/elements/compose-modules/compose-input-module.ts index c989694e660..fa148a27944 100644 --- a/extension/chrome/elements/compose-modules/compose-input-module.ts +++ b/extension/chrome/elements/compose-modules/compose-input-module.ts @@ -245,13 +245,13 @@ export class ComposeInputModule extends ViewModule { /* eslint-disable @typescript-eslint/no-non-null-assertion */ switch (recipient.sendingType) { case 'to': - result.to!.push({ email: recipient.email, name: recipient.name }); + result.to!.push({ email: recipient.email, name: Xss.stripEmojis(recipient.name || '') }); break; case 'cc': - result.cc!.push({ email: recipient.email, name: recipient.name }); + result.cc!.push({ email: recipient.email, name: Xss.stripEmojis(recipient.name || '') }); break; case 'bcc': - result.bcc!.push({ email: recipient.email, name: recipient.name }); + result.bcc!.push({ email: recipient.email, name: Xss.stripEmojis(recipient.name || '') }); break; } /* eslint-enable @typescript-eslint/no-non-null-assertion */ diff --git a/extension/chrome/elements/compose-modules/compose-recipients-module.ts b/extension/chrome/elements/compose-modules/compose-recipients-module.ts index 8163763839c..99ca6da4385 100644 --- a/extension/chrome/elements/compose-modules/compose-recipients-module.ts +++ b/extension/chrome/elements/compose-modules/compose-recipients-module.ts @@ -346,7 +346,7 @@ export class ComposeRecipientsModule extends ViewModule { const recipient = orderedRecipients[processed]; const escapedTitle = Xss.escape(recipient.element.getAttribute('title') || ''); const nameOrEmail = recipient.name || recipient.email || recipient.invalid || ''; - const emailHtml = ``; + const emailHtml = ``; $(emailHtml).insertBefore(rest); // xss-escaped processed++; } @@ -808,7 +808,7 @@ export class ComposeRecipientsModule extends ViewModule { } displayEmail = '
' + Xss.escape(displayEmail) + '
'; if (contact.name) { - ulHtml += '
' + Xss.escape(contact.name) + displayEmail + '
'; + ulHtml += '
' + Xss.stripEmojis(contact.name) + displayEmail + '
'; } else { ulHtml += displayEmail; } @@ -906,7 +906,7 @@ export class ComposeRecipientsModule extends ViewModule { const recipientId = this.generateRecipientId(); const recipientsHtml = `` + - `${Xss.escape(name || '')}` + + `${Xss.stripEmojis(name || '')}` + `${Xss.escape(email || invalid || '')} ${Ui.spinner('green')}`; Xss.sanitizeAppend(container.find('.recipients'), recipientsHtml); const element = document.getElementById(recipientId); @@ -1057,7 +1057,7 @@ export class ComposeRecipientsModule extends ViewModule { } else if (info && info.sortedPubkeys.length) { if (info.info.name) { recipient.name = info.info.name; - $(el).find('.recipient-name').text(Xss.escape(info.info.name)); + $(el).find('.recipient-name').text(Xss.stripEmojis(info.info.name)); } // New logic: // 1. Keys are sorted in a special way. @@ -1098,7 +1098,7 @@ export class ComposeRecipientsModule extends ViewModule { $(el).addClass('no_pgp'); if (info?.info.name) { recipient.name = info.info.name; - $(el).find('.recipient-name').text(Xss.escape(info.info.name)); + $(el).find('.recipient-name').text(Xss.stripEmojis(info.info.name)); } Xss.sanitizePrepend(el, ''); $(el).attr('title', 'Could not verify their encryption setup. You can encrypt the message with a password below. Alternatively, add their pubkey.'); diff --git a/extension/js/common/platform/xss.ts b/extension/js/common/platform/xss.ts index 4e038161e44..5beaf6f4ca0 100644 --- a/extension/js/common/platform/xss.ts +++ b/extension/js/common/platform/xss.ts @@ -230,6 +230,15 @@ export class Xss { return str.replace(/&/g, '&').replace(/"/g, '"').replace(/'/g, ''').replace(//g, '>').replace(/\//g, '/'); }; + public static stripEmojis = (str: string) => { + const emojiRegex = /[\u{1F300}-\u{1F64F}\u{2700}-\u{27B0}\u{23E9}-\u{23FA}\u{2194}-\u{21AA}\u261D\u26A1\u263A-\uD83E\uD83E]/gu; + // regex for removing whitespace other form of whitespaces www.utf8-chartable.de/unicode-utf8-table.pl?start=65024&names=- + const whitespaceRegex = /[\u{FE00}-\u{FE0F}]/gu; + str = str.replace(emojiRegex, ''); + str = str.replace(whitespaceRegex, ''); // + return this.escape(str); + }; + public static htmlUnescape = (str: string) => { // the   at the end is replaced with an actual NBSP character, not a space character. IDE won't show you the difference. Do not change. return str From 063cae68ee68c2ff2b85a081c65073535c87a9cd Mon Sep 17 00:00:00 2001 From: Mart Date: Sun, 23 Jul 2023 18:38:36 +0800 Subject: [PATCH 02/19] add test --- test/source/tests/compose.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index 3a6da930f8c..5b2cc7dbdb6 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -66,6 +66,25 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te }) ); + test( + 'compose - strip emojis in a recipient email address', + testWithBrowser(async (t, browser) => { + const acct = 'flowcrypt.compatibility@gmail.com'; + await BrowserRecipe.setupCommonAcctWithAttester(t, browser, 'compatibility', { + google: { acctAliases: flowcryptCompatibilityAliasList }, + }); + const recipientEmail = 'User ⭐ Name '; + const msgPwd = 'super hard password for the message'; + const subject = 'PWD and pubkey encrypted messages with flowcrypt.com/shared-tenant-fes'; + const composePage = await ComposePageRecipe.openStandalone(t, browser, 'compatibility'); + await ComposePageRecipe.selectFromOption(composePage, acct); + await ComposePageRecipe.fillMsg(composePage, { to: recipientEmail }, subject); + await ComposePageRecipe.sendAndClose(composePage, { password: msgPwd }); + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(1); + // this test is using PwdAndPubkeyEncryptedMessagesWithFlowCryptComApiTestStrategy to check sent result based on subject "PWD and pubkey encrypted messages with flowcrypt.com/shared-tenant-fes" + }) + ); + test( 'compose - check for sender [flowcrypt.compatibility@gmail.com] from a password-protected email', testWithBrowser(async (t, browser) => { From 83eb2ee6fd7c840daa1ac49558aee58a71a99ccc Mon Sep 17 00:00:00 2001 From: martgil Date: Sat, 29 Jul 2023 14:12:56 +0800 Subject: [PATCH 03/19] update compose.ts --- test/source/tests/compose.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index 19c01347953..56ca4f68e70 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -75,13 +75,11 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te }); const recipientEmail = 'User ⭐ Name '; const msgPwd = 'super hard password for the message'; - const subject = 'PWD and pubkey encrypted messages with flowcrypt.com/shared-tenant-fes'; + const subject = 'Strip emojis in display name format'; const composePage = await ComposePageRecipe.openStandalone(t, browser, 'compatibility'); await ComposePageRecipe.selectFromOption(composePage, acct); await ComposePageRecipe.fillMsg(composePage, { to: recipientEmail }, subject); await ComposePageRecipe.sendAndClose(composePage, { password: msgPwd }); - expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(1); - // this test is using PwdAndPubkeyEncryptedMessagesWithFlowCryptComApiTestStrategy to check sent result based on subject "PWD and pubkey encrypted messages with flowcrypt.com/shared-tenant-fes" }) ); From 271aca32811d09316656b05708da8558238ee4e8 Mon Sep 17 00:00:00 2001 From: martgil Date: Mon, 31 Jul 2023 17:55:08 +0800 Subject: [PATCH 04/19] store recipient info to recipientObj --- .../elements/compose-modules/compose-input-module.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-input-module.ts b/extension/chrome/elements/compose-modules/compose-input-module.ts index fa148a27944..d72b495e7c8 100644 --- a/extension/chrome/elements/compose-modules/compose-input-module.ts +++ b/extension/chrome/elements/compose-modules/compose-input-module.ts @@ -243,15 +243,16 @@ export class ComposeInputModule extends ViewModule { const result: ParsedRecipients = { to: [], cc: [], bcc: [] }; for (const recipient of recipients) { /* eslint-disable @typescript-eslint/no-non-null-assertion */ + const recipientObj = { email: recipient.email, name: Xss.stripEmojis(recipient.name || '') }; switch (recipient.sendingType) { case 'to': - result.to!.push({ email: recipient.email, name: Xss.stripEmojis(recipient.name || '') }); + result.to!.push(recipientObj); break; case 'cc': - result.cc!.push({ email: recipient.email, name: Xss.stripEmojis(recipient.name || '') }); + result.cc!.push(recipientObj); break; case 'bcc': - result.bcc!.push({ email: recipient.email, name: Xss.stripEmojis(recipient.name || '') }); + result.bcc!.push(recipientObj); break; } /* eslint-enable @typescript-eslint/no-non-null-assertion */ From b3306c622f7c17d18c41c9a2ae25e672114dd219 Mon Sep 17 00:00:00 2001 From: martgil Date: Mon, 31 Jul 2023 18:12:06 +0800 Subject: [PATCH 05/19] add emojiRegex lib --- extension/js/common/platform/xss.ts | 7 ++++--- package-lock.json | 13 ++++++++++--- package.json | 1 + tsconfig.json | 3 ++- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/extension/js/common/platform/xss.ts b/extension/js/common/platform/xss.ts index 5beaf6f4ca0..be5eb2c82a2 100644 --- a/extension/js/common/platform/xss.ts +++ b/extension/js/common/platform/xss.ts @@ -4,10 +4,11 @@ import * as DOMPurify from 'dompurify'; +import emojiRegex from 'emoji-regex'; + import { checkValidURL, CID_PATTERN, Str } from '../core/common.js'; export type SanitizeImgHandling = 'IMG-DEL' | 'IMG-KEEP' | 'IMG-TO-PLAIN-TEXT'; - /** * This class is in platform/ folder because most of it depends on platform specific code * - in browser the implementation uses DOMPurify @@ -231,10 +232,10 @@ export class Xss { }; public static stripEmojis = (str: string) => { - const emojiRegex = /[\u{1F300}-\u{1F64F}\u{2700}-\u{27B0}\u{23E9}-\u{23FA}\u{2194}-\u{21AA}\u261D\u26A1\u263A-\uD83E\uD83E]/gu; + const emojiPattern = emojiRegex(); // regex for removing whitespace other form of whitespaces www.utf8-chartable.de/unicode-utf8-table.pl?start=65024&names=- const whitespaceRegex = /[\u{FE00}-\u{FE0F}]/gu; - str = str.replace(emojiRegex, ''); + str = str.replace(emojiPattern, ''); str = str.replace(whitespaceRegex, ''); // return this.escape(str); }; diff --git a/package-lock.json b/package-lock.json index 4c43eceb168..4180edc7040 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,6 +40,7 @@ "chai": "4.3.7", "chai-as-promised": "7.1.1", "del": "7.0.0", + "emoji-regex": "^10.2.1", "eslint": "^8.46.0", "eslint-config-prettier": "^8.8.0", "eslint-plugin-header": "3.1.1", @@ -4385,9 +4386,9 @@ } }, "node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.2.1.tgz", + "integrity": "sha512-97g6QgOk8zlDRdgq1WxwgTMgEWGVAQvB5Fdpgc1MkNy56la5SKP9GsMXKDOdqwn90/41a8yPwIGk1Y6WVbeMQA==", "dev": true }, "node_modules/encoding-japanese": { @@ -11486,6 +11487,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/string-width/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true + }, "node_modules/string.prototype.trim": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.7.tgz", diff --git a/package.json b/package.json index d5ebc088f57..fb04ba638de 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "chai": "4.3.7", "chai-as-promised": "7.1.1", "del": "7.0.0", + "emoji-regex": "^10.2.1", "eslint": "^8.46.0", "eslint-config-prettier": "^8.8.0", "eslint-plugin-header": "3.1.1", diff --git a/tsconfig.json b/tsconfig.json index 23f761b6a12..7e670c5d01c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -30,7 +30,8 @@ "zxcvbn": ["lib/zxcvbn.js", "COMMENT"], "emailjs-mime-parser": ["lib/emailjs/emailjs-mime-parser.js", "COMMENT"], "emailjs-mime-builder": ["lib/emailjs/emailjs-mime-builder.js", "COMMENT"], - "node-forge": ["lib/forge.js", "COMMENT"] + "node-forge": ["lib/forge.js", "COMMENT"], + "emoji-regex": ["../node_modules/emoji-regex/index.d.ts"] }, "typeRoots": ["./extension/js/common/core/types", "./extension/types"] }, From 04f8de1e37f47199ce5cb22068a07b7b097eaa87 Mon Sep 17 00:00:00 2001 From: martgil Date: Mon, 31 Jul 2023 19:35:54 +0800 Subject: [PATCH 06/19] update reference --- extension/js/common/platform/xss.ts | 3 +++ tsconfig.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/extension/js/common/platform/xss.ts b/extension/js/common/platform/xss.ts index be5eb2c82a2..4fb73903e99 100644 --- a/extension/js/common/platform/xss.ts +++ b/extension/js/common/platform/xss.ts @@ -1,5 +1,8 @@ /* ©️ 2016 - present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com */ +// eslint-disable-next-line @typescript-eslint/triple-slash-reference +/// + 'use strict'; import * as DOMPurify from 'dompurify'; diff --git a/tsconfig.json b/tsconfig.json index 7e670c5d01c..c9a3b66fb89 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -31,7 +31,7 @@ "emailjs-mime-parser": ["lib/emailjs/emailjs-mime-parser.js", "COMMENT"], "emailjs-mime-builder": ["lib/emailjs/emailjs-mime-builder.js", "COMMENT"], "node-forge": ["lib/forge.js", "COMMENT"], - "emoji-regex": ["../node_modules/emoji-regex/index.d.ts"] + "emoji-regex": ["../node_modules/emoji-regex/index.d.ts", "../node_modules/emoji-regex/index.js", "COMMENT"] }, "typeRoots": ["./extension/js/common/core/types", "./extension/types"] }, From 140bfc12c58a527b16b951a7ac7b4b015d5da7d0 Mon Sep 17 00:00:00 2001 From: martgil Date: Wed, 2 Aug 2023 09:47:12 +0800 Subject: [PATCH 07/19] github ci test --- extension/js/common/platform/xss.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/extension/js/common/platform/xss.ts b/extension/js/common/platform/xss.ts index 4fb73903e99..bb02d6aed46 100644 --- a/extension/js/common/platform/xss.ts +++ b/extension/js/common/platform/xss.ts @@ -235,6 +235,7 @@ export class Xss { }; public static stripEmojis = (str: string) => { + console.log('ci test'); const emojiPattern = emojiRegex(); // regex for removing whitespace other form of whitespaces www.utf8-chartable.de/unicode-utf8-table.pl?start=65024&names=- const whitespaceRegex = /[\u{FE00}-\u{FE0F}]/gu; From 41571175c4d691537cf34913c54d9d3cb360f633 Mon Sep 17 00:00:00 2001 From: martgil Date: Wed, 2 Aug 2023 09:47:55 +0800 Subject: [PATCH 08/19] remove github ci test --- extension/js/common/platform/xss.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/extension/js/common/platform/xss.ts b/extension/js/common/platform/xss.ts index bb02d6aed46..4fb73903e99 100644 --- a/extension/js/common/platform/xss.ts +++ b/extension/js/common/platform/xss.ts @@ -235,7 +235,6 @@ export class Xss { }; public static stripEmojis = (str: string) => { - console.log('ci test'); const emojiPattern = emojiRegex(); // regex for removing whitespace other form of whitespaces www.utf8-chartable.de/unicode-utf8-table.pl?start=65024&names=- const whitespaceRegex = /[\u{FE00}-\u{FE0F}]/gu; From c987f0cc8b4d2cc0c15b7cf7823b360d29af9678 Mon Sep 17 00:00:00 2001 From: martgil Date: Wed, 2 Aug 2023 14:48:14 +0800 Subject: [PATCH 09/19] add webpack configuration emoji-regex --- .../{webpack.config.js => webpack.config1.js} | 0 conf/webpack.config2.js | 22 +++++++++++++++++++ extension/chrome/dev/ci_unit_test.htm | 1 + extension/chrome/elements/compose.htm | 1 + scripts/build.sh | 3 ++- 5 files changed, 26 insertions(+), 1 deletion(-) rename conf/{webpack.config.js => webpack.config1.js} (100%) create mode 100644 conf/webpack.config2.js diff --git a/conf/webpack.config.js b/conf/webpack.config1.js similarity index 100% rename from conf/webpack.config.js rename to conf/webpack.config1.js diff --git a/conf/webpack.config2.js b/conf/webpack.config2.js new file mode 100644 index 00000000000..f8198463d15 --- /dev/null +++ b/conf/webpack.config2.js @@ -0,0 +1,22 @@ +//webpack.config.js +//bundle the web version of @openpgp/web-stream-tools for content script +const path = require('path'); + +module.exports = { + mode: 'production', + entry: ['../build/generic-extension-wip/lib/emoji-regex.js'], + output: { + library: { + name: 'emojiRegex', + type: 'var', + }, + path: path.resolve('../build/generic-extension-wip/lib'), + filename: 'emoji_regex_web.js', // <--- Will be compiled to this single file + }, + resolve: { + fallback: { + stream: false, + }, + extensions: ['.js'], + }, +}; diff --git a/extension/chrome/dev/ci_unit_test.htm b/extension/chrome/dev/ci_unit_test.htm index cc7e3b3c137..7cd2878b347 100644 --- a/extension/chrome/dev/ci_unit_test.htm +++ b/extension/chrome/dev/ci_unit_test.htm @@ -26,6 +26,7 @@

loading..

+ diff --git a/extension/chrome/elements/compose.htm b/extension/chrome/elements/compose.htm index a9997fe3344..fc1193de0ce 100644 --- a/extension/chrome/elements/compose.htm +++ b/extension/chrome/elements/compose.htm @@ -244,6 +244,7 @@

New Secure Message

+ diff --git a/scripts/build.sh b/scripts/build.sh index 6cb799a5ea5..b1415252649 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -55,6 +55,7 @@ copy_dependencies() { cp node_modules/bootstrap/dist/js/bootstrap.min.js $OUTPUT_DIRECTORY/lib/bootstrap/bootstrap.min.js cp node_modules/bootstrap/dist/css/bootstrap.min.css $OUTPUT_DIRECTORY/lib/bootstrap/bootstrap.min.css cp node_modules/@openpgp/web-stream-tools/lib/*.js $OUTPUT_DIRECTORY/lib/streams + cp node_modules/emoji-regex/index.js $OUTPUT_DIRECTORY/lib/emoji-regex.js } # Function to run a regex replace command with sed @@ -128,7 +129,7 @@ main() { apply_regex_replace $ISUINT8ARRAY_REGEX3 $OPENPGP_FILE # bundle web-stream-tools as Stream var for the content script - ( cd conf && npx webpack ) & pids+=($!) + ( cd conf && npx webpack --config webpack.config1.js && npx webpack --config webpack.config2.js ) & pids+=($!) for pid in "${pids[@]}"; do wait "$pid" || exit 1; done # to update node-forge library, which is missing the non-minified version in dist, we have to build it manually From f06e35835640bf90f085156a22a8daf6559f797d Mon Sep 17 00:00:00 2001 From: martgil Date: Wed, 27 Sep 2023 15:03:14 +0800 Subject: [PATCH 10/19] remove emoji-regex package --- .../{webpack.config1.js => webpack.config.js} | 0 conf/webpack.config2.js | 22 ------------------- extension/js/common/platform/xss.ts | 9 +------- package-lock.json | 19 +++++----------- package.json | 1 - scripts/build.sh | 3 +-- tsconfig.json | 3 +-- 7 files changed, 9 insertions(+), 48 deletions(-) rename conf/{webpack.config1.js => webpack.config.js} (100%) delete mode 100644 conf/webpack.config2.js diff --git a/conf/webpack.config1.js b/conf/webpack.config.js similarity index 100% rename from conf/webpack.config1.js rename to conf/webpack.config.js diff --git a/conf/webpack.config2.js b/conf/webpack.config2.js deleted file mode 100644 index f8198463d15..00000000000 --- a/conf/webpack.config2.js +++ /dev/null @@ -1,22 +0,0 @@ -//webpack.config.js -//bundle the web version of @openpgp/web-stream-tools for content script -const path = require('path'); - -module.exports = { - mode: 'production', - entry: ['../build/generic-extension-wip/lib/emoji-regex.js'], - output: { - library: { - name: 'emojiRegex', - type: 'var', - }, - path: path.resolve('../build/generic-extension-wip/lib'), - filename: 'emoji_regex_web.js', // <--- Will be compiled to this single file - }, - resolve: { - fallback: { - stream: false, - }, - extensions: ['.js'], - }, -}; diff --git a/extension/js/common/platform/xss.ts b/extension/js/common/platform/xss.ts index 4fb73903e99..d3f55661eba 100644 --- a/extension/js/common/platform/xss.ts +++ b/extension/js/common/platform/xss.ts @@ -1,14 +1,11 @@ /* ©️ 2016 - present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com */ // eslint-disable-next-line @typescript-eslint/triple-slash-reference -/// 'use strict'; import * as DOMPurify from 'dompurify'; -import emojiRegex from 'emoji-regex'; - import { checkValidURL, CID_PATTERN, Str } from '../core/common.js'; export type SanitizeImgHandling = 'IMG-DEL' | 'IMG-KEEP' | 'IMG-TO-PLAIN-TEXT'; @@ -235,11 +232,7 @@ export class Xss { }; public static stripEmojis = (str: string) => { - const emojiPattern = emojiRegex(); - // regex for removing whitespace other form of whitespaces www.utf8-chartable.de/unicode-utf8-table.pl?start=65024&names=- - const whitespaceRegex = /[\u{FE00}-\u{FE0F}]/gu; - str = str.replace(emojiPattern, ''); - str = str.replace(whitespaceRegex, ''); // + str = str.replace(/(?![*#0-9]+)[\p{Emoji}\p{Emoji_Modifier}\p{Emoji_Component}\p{Emoji_Modifier_Base}\p{Emoji_Presentation}]/gu, ''); return this.escape(str); }; diff --git a/package-lock.json b/package-lock.json index df9fc6c7835..1db54893372 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,7 +39,6 @@ "ava": "5.3.1", "chai": "4.3.8", "chai-as-promised": "7.1.1", - "emoji-regex": "^10.2.1", "eslint": "^8.48.0", "eslint-config-prettier": "^9.0.0", "eslint-plugin-header": "3.1.1", @@ -4239,12 +4238,6 @@ "url": "https://github.com/sindresorhus/emittery?sponsor=1" } }, - "node_modules/emoji-regex": { - "version": "10.2.1", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.2.1.tgz", - "integrity": "sha512-97g6QgOk8zlDRdgq1WxwgTMgEWGVAQvB5Fdpgc1MkNy56la5SKP9GsMXKDOdqwn90/41a8yPwIGk1Y6WVbeMQA==", - "dev": true - }, "node_modules/encoding-japanese": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encoding-japanese/-/encoding-japanese-2.0.0.tgz", @@ -10892,12 +10885,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/string-width/node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "dev": true - }, "node_modules/string-width-cjs": { "name": "string-width", "version": "4.2.3", @@ -10949,6 +10936,12 @@ "node": ">=8" } }, + "node_modules/string-width/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true + }, "node_modules/string.prototype.trim": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.8.tgz", diff --git a/package.json b/package.json index f22cf0056b3..f4278c014aa 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,6 @@ "ava": "5.3.1", "chai": "4.3.8", "chai-as-promised": "7.1.1", - "emoji-regex": "^10.2.1", "eslint": "^8.48.0", "eslint-config-prettier": "^9.0.0", "eslint-plugin-header": "3.1.1", diff --git a/scripts/build.sh b/scripts/build.sh index b1415252649..bf93743b3df 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -55,7 +55,6 @@ copy_dependencies() { cp node_modules/bootstrap/dist/js/bootstrap.min.js $OUTPUT_DIRECTORY/lib/bootstrap/bootstrap.min.js cp node_modules/bootstrap/dist/css/bootstrap.min.css $OUTPUT_DIRECTORY/lib/bootstrap/bootstrap.min.css cp node_modules/@openpgp/web-stream-tools/lib/*.js $OUTPUT_DIRECTORY/lib/streams - cp node_modules/emoji-regex/index.js $OUTPUT_DIRECTORY/lib/emoji-regex.js } # Function to run a regex replace command with sed @@ -129,7 +128,7 @@ main() { apply_regex_replace $ISUINT8ARRAY_REGEX3 $OPENPGP_FILE # bundle web-stream-tools as Stream var for the content script - ( cd conf && npx webpack --config webpack.config1.js && npx webpack --config webpack.config2.js ) & pids+=($!) + ( cd conf && npx webpack --config webpack.config.js ) & pids+=($!) for pid in "${pids[@]}"; do wait "$pid" || exit 1; done # to update node-forge library, which is missing the non-minified version in dist, we have to build it manually diff --git a/tsconfig.json b/tsconfig.json index c9a3b66fb89..23f761b6a12 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -30,8 +30,7 @@ "zxcvbn": ["lib/zxcvbn.js", "COMMENT"], "emailjs-mime-parser": ["lib/emailjs/emailjs-mime-parser.js", "COMMENT"], "emailjs-mime-builder": ["lib/emailjs/emailjs-mime-builder.js", "COMMENT"], - "node-forge": ["lib/forge.js", "COMMENT"], - "emoji-regex": ["../node_modules/emoji-regex/index.d.ts", "../node_modules/emoji-regex/index.js", "COMMENT"] + "node-forge": ["lib/forge.js", "COMMENT"] }, "typeRoots": ["./extension/js/common/core/types", "./extension/types"] }, From fa17ce5abe8ee53ab09ae1ec9d3a2388b48e20e5 Mon Sep 17 00:00:00 2001 From: martgil Date: Wed, 27 Sep 2023 17:55:08 +0800 Subject: [PATCH 11/19] add Intl.Segmenter polyfill --- conf/tsconfig.content_scripts.json | 3 ++- conf/webpack.config_intl_segmenter.js | 20 +++++++++++++++++ extension/chrome/dev/ci_unit_test.htm | 4 +++- extension/chrome/elements/compose.htm | 4 +++- extension/js/common/platform/xss.ts | 10 ++++++++- package-lock.json | 31 +++++++++++++++++++++++++++ package.json | 1 + scripts/build.sh | 5 ++++- tsconfig.json | 6 ++++++ 9 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 conf/webpack.config_intl_segmenter.js diff --git a/conf/tsconfig.content_scripts.json b/conf/tsconfig.content_scripts.json index b5bcb994288..1cfe7a87a9a 100644 --- a/conf/tsconfig.content_scripts.json +++ b/conf/tsconfig.content_scripts.json @@ -18,7 +18,8 @@ "paths": { "dompurify": ["types/purify.d.ts"], "openpgp": ["../node_modules/openpgp/openpgp.d.ts"], - "@openpgp/web-stream-tools": ["../node_modules/@openpgp/web-stream-tools/types/index.v4.9.d.ts"] + "@openpgp/web-stream-tools": ["../node_modules/@openpgp/web-stream-tools/types/index.v4.9.d.ts"], + "@formatjs/intl-segmenter": ["../node_modules/@formatjs/intl-segmenter/lib/src/segmenter.d.ts"] }, "typeRoots": ["../extension/types", "../extension/js/common/core/types"] }, diff --git a/conf/webpack.config_intl_segmenter.js b/conf/webpack.config_intl_segmenter.js new file mode 100644 index 00000000000..58a199af3d9 --- /dev/null +++ b/conf/webpack.config_intl_segmenter.js @@ -0,0 +1,20 @@ +const path = require('path'); + +module.exports = { + mode: 'production', + entry: ['../build/generic-extension-wip/lib/intl_segmenter.js'], + output: { + library: { + name: 'Segmenter', + type: 'var', + }, + path: path.resolve('../build/generic-extension-wip/lib'), + filename: 'intl_segmenter.js', // <--- Will be compiled to this single file + }, + resolve: { + fallback: { + stream: false, + }, + extensions: ['.js'], + }, +}; diff --git a/extension/chrome/dev/ci_unit_test.htm b/extension/chrome/dev/ci_unit_test.htm index 7cd2878b347..b78ec4694df 100644 --- a/extension/chrome/dev/ci_unit_test.htm +++ b/extension/chrome/dev/ci_unit_test.htm @@ -26,7 +26,9 @@

loading..

- + + + diff --git a/extension/chrome/elements/compose.htm b/extension/chrome/elements/compose.htm index fc1193de0ce..5e7263fcdd9 100644 --- a/extension/chrome/elements/compose.htm +++ b/extension/chrome/elements/compose.htm @@ -244,7 +244,9 @@

New Secure Message

- + + + diff --git a/extension/js/common/platform/xss.ts b/extension/js/common/platform/xss.ts index d3f55661eba..6e1202b548b 100644 --- a/extension/js/common/platform/xss.ts +++ b/extension/js/common/platform/xss.ts @@ -1,11 +1,14 @@ /* ©️ 2016 - present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com */ // eslint-disable-next-line @typescript-eslint/triple-slash-reference +/// 'use strict'; import * as DOMPurify from 'dompurify'; +import { Segmenter } from '@formatjs/intl-segmenter'; + import { checkValidURL, CID_PATTERN, Str } from '../core/common.js'; export type SanitizeImgHandling = 'IMG-DEL' | 'IMG-KEEP' | 'IMG-TO-PLAIN-TEXT'; @@ -232,7 +235,12 @@ export class Xss { }; public static stripEmojis = (str: string) => { - str = str.replace(/(?![*#0-9]+)[\p{Emoji}\p{Emoji_Modifier}\p{Emoji_Component}\p{Emoji_Modifier_Base}\p{Emoji_Presentation}]/gu, ''); + if (Intl !== undefined && typeof Segmenter === 'function') { + const segmenter = new Segmenter('en', {}); + str = [...segmenter.segment(str)].map(x => x?.segment).join(''); + } else { + str = str.replace(/(?![*#0-9]+)[\p{Emoji}\p{Emoji_Modifier}\p{Emoji_Component}\p{Emoji_Modifier_Base}\p{Emoji_Presentation}]/gu, ''); + } return this.escape(str); }; diff --git a/package-lock.json b/package-lock.json index 1db54893372..0a90d8f1d95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,7 @@ "zxcvbn": "4.4.2" }, "devDependencies": { + "@formatjs/intl-segmenter": "^11.4.4", "@openpgp/web-stream-tools": "^0.0.13", "@types/chai": "4.3.6", "@types/chai-as-promised": "7.1.6", @@ -467,6 +468,36 @@ "npm": ">=7.0.0" } }, + "node_modules/@formatjs/ecma402-abstract": { + "version": "1.17.2", + "resolved": "https://registry.npmjs.org/@formatjs/ecma402-abstract/-/ecma402-abstract-1.17.2.tgz", + "integrity": "sha512-k2mTh0m+IV1HRdU0xXM617tSQTi53tVR2muvYOsBeYcUgEAyxV1FOC7Qj279th3fBVQ+Dj6muvNJZcHSPNdbKg==", + "dev": true, + "dependencies": { + "@formatjs/intl-localematcher": "0.4.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/intl-localematcher": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@formatjs/intl-localematcher/-/intl-localematcher-0.4.2.tgz", + "integrity": "sha512-BGdtJFmaNJy5An/Zan4OId/yR9Ih1OojFjcduX/xOvq798OgWSyDtd6Qd5jqJXwJs1ipe4Fxu9+cshic5Ox2tA==", + "dev": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/intl-segmenter": { + "version": "11.4.4", + "resolved": "https://registry.npmjs.org/@formatjs/intl-segmenter/-/intl-segmenter-11.4.4.tgz", + "integrity": "sha512-dtSNa7DO3/i3Qro+U50Xf82KlKs1XfLOPcDb2YNpaNuqrcJD1V70sW7QbPPR56lTuLnHUS7js3ymEhxKSaZtfg==", + "dev": true, + "dependencies": { + "@formatjs/ecma402-abstract": "1.17.2", + "@formatjs/intl-localematcher": "0.4.2", + "tslib": "^2.4.0" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.11", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.11.tgz", diff --git a/package.json b/package.json index f4278c014aa..0a001c63bbe 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "graceful-fs": "4.1.13" }, "devDependencies": { + "@formatjs/intl-segmenter": "^11.4.4", "@openpgp/web-stream-tools": "^0.0.13", "@types/chai": "4.3.6", "@types/chai-as-promised": "7.1.6", diff --git a/scripts/build.sh b/scripts/build.sh index bf93743b3df..898e4371384 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -55,6 +55,9 @@ copy_dependencies() { cp node_modules/bootstrap/dist/js/bootstrap.min.js $OUTPUT_DIRECTORY/lib/bootstrap/bootstrap.min.js cp node_modules/bootstrap/dist/css/bootstrap.min.css $OUTPUT_DIRECTORY/lib/bootstrap/bootstrap.min.css cp node_modules/@openpgp/web-stream-tools/lib/*.js $OUTPUT_DIRECTORY/lib/streams + cp node_modules/@formatjs/intl-segmenter/lib/src/segmenter.js $OUTPUT_DIRECTORY/lib/intl_segmenter.js + cp node_modules/@formatjs/intl-segmenter/lib/src/segmentation-utils.js $OUTPUT_DIRECTORY/lib/segmentation-utils.js + cp node_modules/@formatjs/intl-segmenter/lib/src/cldr-segmentation-rules.generated.js $OUTPUT_DIRECTORY/lib/cldr-segmentation-rules.generated.js } # Function to run a regex replace command with sed @@ -128,7 +131,7 @@ main() { apply_regex_replace $ISUINT8ARRAY_REGEX3 $OPENPGP_FILE # bundle web-stream-tools as Stream var for the content script - ( cd conf && npx webpack --config webpack.config.js ) & pids+=($!) + ( cd conf && npx webpack --config webpack.config.js && npx webpack --config webpack.config_intl_segmenter.js ) & pids+=($!) for pid in "${pids[@]}"; do wait "$pid" || exit 1; done # to update node-forge library, which is missing the non-minified version in dist, we have to build it manually diff --git a/tsconfig.json b/tsconfig.json index 23f761b6a12..f36a352458a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,6 +22,12 @@ "sweetalert2": ["lib/sweetalert2.js", "COMMENT"], "openpgp": ["../node_modules/openpgp/openpgp.d.js", "lib/openpgp.js", "COMMENT"], "@openpgp/web-stream-tools": ["../node_modules/@openpgp/web-stream-tools/types/index.v4.9.d.ts", "lib/streams/streams.js"], + "@formatjs/intl-segmenter": [ + "../node_modules/@formatjs/intl-segmenter/lib/src/segmenter.d.ts", + "../node_modules/@formatjs/intl-segmenter/lib/src/segmenter.js", + "lib/segmenter.js", + "COMMENT" + ], "dompurify": ["types/purify.d.ts", "lib/purify.js", "COMMENT"], "fine-uploader": ["lib/fine-uploader.js", "COMMENT"], "clipboard": ["lib/clipboard.js", "COMMENT"], From 6a21f97a876364f091b22349671c97fa2fccc318 Mon Sep 17 00:00:00 2001 From: martgil Date: Wed, 27 Sep 2023 18:17:15 +0800 Subject: [PATCH 12/19] remove Intl.Segmenter polyfill --- conf/tsconfig.content_scripts.json | 3 +-- conf/webpack.config_intl_segmenter.js | 20 ----------------- extension/chrome/dev/ci_unit_test.htm | 3 --- extension/chrome/elements/compose.htm | 3 --- extension/js/common/platform/xss.ts | 13 +++++------ package-lock.json | 31 --------------------------- package.json | 1 - scripts/build.sh | 5 +---- tsconfig.json | 6 ------ 9 files changed, 7 insertions(+), 78 deletions(-) delete mode 100644 conf/webpack.config_intl_segmenter.js diff --git a/conf/tsconfig.content_scripts.json b/conf/tsconfig.content_scripts.json index 1cfe7a87a9a..b5bcb994288 100644 --- a/conf/tsconfig.content_scripts.json +++ b/conf/tsconfig.content_scripts.json @@ -18,8 +18,7 @@ "paths": { "dompurify": ["types/purify.d.ts"], "openpgp": ["../node_modules/openpgp/openpgp.d.ts"], - "@openpgp/web-stream-tools": ["../node_modules/@openpgp/web-stream-tools/types/index.v4.9.d.ts"], - "@formatjs/intl-segmenter": ["../node_modules/@formatjs/intl-segmenter/lib/src/segmenter.d.ts"] + "@openpgp/web-stream-tools": ["../node_modules/@openpgp/web-stream-tools/types/index.v4.9.d.ts"] }, "typeRoots": ["../extension/types", "../extension/js/common/core/types"] }, diff --git a/conf/webpack.config_intl_segmenter.js b/conf/webpack.config_intl_segmenter.js deleted file mode 100644 index 58a199af3d9..00000000000 --- a/conf/webpack.config_intl_segmenter.js +++ /dev/null @@ -1,20 +0,0 @@ -const path = require('path'); - -module.exports = { - mode: 'production', - entry: ['../build/generic-extension-wip/lib/intl_segmenter.js'], - output: { - library: { - name: 'Segmenter', - type: 'var', - }, - path: path.resolve('../build/generic-extension-wip/lib'), - filename: 'intl_segmenter.js', // <--- Will be compiled to this single file - }, - resolve: { - fallback: { - stream: false, - }, - extensions: ['.js'], - }, -}; diff --git a/extension/chrome/dev/ci_unit_test.htm b/extension/chrome/dev/ci_unit_test.htm index b78ec4694df..cc7e3b3c137 100644 --- a/extension/chrome/dev/ci_unit_test.htm +++ b/extension/chrome/dev/ci_unit_test.htm @@ -26,9 +26,6 @@

loading..

- - - diff --git a/extension/chrome/elements/compose.htm b/extension/chrome/elements/compose.htm index 5e7263fcdd9..a9997fe3344 100644 --- a/extension/chrome/elements/compose.htm +++ b/extension/chrome/elements/compose.htm @@ -244,9 +244,6 @@

New Secure Message

- - - diff --git a/extension/js/common/platform/xss.ts b/extension/js/common/platform/xss.ts index 6e1202b548b..9be6f4d20f2 100644 --- a/extension/js/common/platform/xss.ts +++ b/extension/js/common/platform/xss.ts @@ -1,14 +1,11 @@ /* ©️ 2016 - present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com */ -// eslint-disable-next-line @typescript-eslint/triple-slash-reference -/// +/// 'use strict'; import * as DOMPurify from 'dompurify'; -import { Segmenter } from '@formatjs/intl-segmenter'; - import { checkValidURL, CID_PATTERN, Str } from '../core/common.js'; export type SanitizeImgHandling = 'IMG-DEL' | 'IMG-KEEP' | 'IMG-TO-PLAIN-TEXT'; @@ -55,6 +52,7 @@ export class Xss { private static FORBID_ATTR = ['background']; private static HREF_REGEX_CACHE: RegExp | undefined; private static FORBID_CSS_STYLE = /z-index:[^;]+;|position:[^;]+;|background[^;]+;/g; + private static EMOJI_REGEX = /(?![*#0-9]+)[\p{Emoji}\p{Emoji_Modifier}\p{Emoji_Component}\p{Emoji_Modifier_Base}\p{Emoji_Presentation}]/gu; public static sanitizeRender = (selector: string | HTMLElement | JQuery, dirtyHtml: string) => { // browser-only (not on node) @@ -235,11 +233,10 @@ export class Xss { }; public static stripEmojis = (str: string) => { - if (Intl !== undefined && typeof Segmenter === 'function') { - const segmenter = new Segmenter('en', {}); - str = [...segmenter.segment(str)].map(x => x?.segment).join(''); + if (Intl !== undefined && typeof Intl.Segmenter === 'function') { + str = [...new Intl.Segmenter().segment(str)].map(x => x?.segment).join(''); } else { - str = str.replace(/(?![*#0-9]+)[\p{Emoji}\p{Emoji_Modifier}\p{Emoji_Component}\p{Emoji_Modifier_Base}\p{Emoji_Presentation}]/gu, ''); + str = str.replace(Xss.EMOJI_REGEX, ''); } return this.escape(str); }; diff --git a/package-lock.json b/package-lock.json index 0a90d8f1d95..1db54893372 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,7 +27,6 @@ "zxcvbn": "4.4.2" }, "devDependencies": { - "@formatjs/intl-segmenter": "^11.4.4", "@openpgp/web-stream-tools": "^0.0.13", "@types/chai": "4.3.6", "@types/chai-as-promised": "7.1.6", @@ -468,36 +467,6 @@ "npm": ">=7.0.0" } }, - "node_modules/@formatjs/ecma402-abstract": { - "version": "1.17.2", - "resolved": "https://registry.npmjs.org/@formatjs/ecma402-abstract/-/ecma402-abstract-1.17.2.tgz", - "integrity": "sha512-k2mTh0m+IV1HRdU0xXM617tSQTi53tVR2muvYOsBeYcUgEAyxV1FOC7Qj279th3fBVQ+Dj6muvNJZcHSPNdbKg==", - "dev": true, - "dependencies": { - "@formatjs/intl-localematcher": "0.4.2", - "tslib": "^2.4.0" - } - }, - "node_modules/@formatjs/intl-localematcher": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@formatjs/intl-localematcher/-/intl-localematcher-0.4.2.tgz", - "integrity": "sha512-BGdtJFmaNJy5An/Zan4OId/yR9Ih1OojFjcduX/xOvq798OgWSyDtd6Qd5jqJXwJs1ipe4Fxu9+cshic5Ox2tA==", - "dev": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@formatjs/intl-segmenter": { - "version": "11.4.4", - "resolved": "https://registry.npmjs.org/@formatjs/intl-segmenter/-/intl-segmenter-11.4.4.tgz", - "integrity": "sha512-dtSNa7DO3/i3Qro+U50Xf82KlKs1XfLOPcDb2YNpaNuqrcJD1V70sW7QbPPR56lTuLnHUS7js3ymEhxKSaZtfg==", - "dev": true, - "dependencies": { - "@formatjs/ecma402-abstract": "1.17.2", - "@formatjs/intl-localematcher": "0.4.2", - "tslib": "^2.4.0" - } - }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.11", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.11.tgz", diff --git a/package.json b/package.json index 0a001c63bbe..f4278c014aa 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,6 @@ "graceful-fs": "4.1.13" }, "devDependencies": { - "@formatjs/intl-segmenter": "^11.4.4", "@openpgp/web-stream-tools": "^0.0.13", "@types/chai": "4.3.6", "@types/chai-as-promised": "7.1.6", diff --git a/scripts/build.sh b/scripts/build.sh index 898e4371384..bf93743b3df 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -55,9 +55,6 @@ copy_dependencies() { cp node_modules/bootstrap/dist/js/bootstrap.min.js $OUTPUT_DIRECTORY/lib/bootstrap/bootstrap.min.js cp node_modules/bootstrap/dist/css/bootstrap.min.css $OUTPUT_DIRECTORY/lib/bootstrap/bootstrap.min.css cp node_modules/@openpgp/web-stream-tools/lib/*.js $OUTPUT_DIRECTORY/lib/streams - cp node_modules/@formatjs/intl-segmenter/lib/src/segmenter.js $OUTPUT_DIRECTORY/lib/intl_segmenter.js - cp node_modules/@formatjs/intl-segmenter/lib/src/segmentation-utils.js $OUTPUT_DIRECTORY/lib/segmentation-utils.js - cp node_modules/@formatjs/intl-segmenter/lib/src/cldr-segmentation-rules.generated.js $OUTPUT_DIRECTORY/lib/cldr-segmentation-rules.generated.js } # Function to run a regex replace command with sed @@ -131,7 +128,7 @@ main() { apply_regex_replace $ISUINT8ARRAY_REGEX3 $OPENPGP_FILE # bundle web-stream-tools as Stream var for the content script - ( cd conf && npx webpack --config webpack.config.js && npx webpack --config webpack.config_intl_segmenter.js ) & pids+=($!) + ( cd conf && npx webpack --config webpack.config.js ) & pids+=($!) for pid in "${pids[@]}"; do wait "$pid" || exit 1; done # to update node-forge library, which is missing the non-minified version in dist, we have to build it manually diff --git a/tsconfig.json b/tsconfig.json index f36a352458a..23f761b6a12 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,12 +22,6 @@ "sweetalert2": ["lib/sweetalert2.js", "COMMENT"], "openpgp": ["../node_modules/openpgp/openpgp.d.js", "lib/openpgp.js", "COMMENT"], "@openpgp/web-stream-tools": ["../node_modules/@openpgp/web-stream-tools/types/index.v4.9.d.ts", "lib/streams/streams.js"], - "@formatjs/intl-segmenter": [ - "../node_modules/@formatjs/intl-segmenter/lib/src/segmenter.d.ts", - "../node_modules/@formatjs/intl-segmenter/lib/src/segmenter.js", - "lib/segmenter.js", - "COMMENT" - ], "dompurify": ["types/purify.d.ts", "lib/purify.js", "COMMENT"], "fine-uploader": ["lib/fine-uploader.js", "COMMENT"], "clipboard": ["lib/clipboard.js", "COMMENT"], From 10fe721e1b297513c68bedb9bf7e2b5bf4219273 Mon Sep 17 00:00:00 2001 From: martgil Date: Wed, 27 Sep 2023 19:42:58 +0800 Subject: [PATCH 13/19] revert changes on build.sh --- scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build.sh b/scripts/build.sh index bf93743b3df..6cb799a5ea5 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -128,7 +128,7 @@ main() { apply_regex_replace $ISUINT8ARRAY_REGEX3 $OPENPGP_FILE # bundle web-stream-tools as Stream var for the content script - ( cd conf && npx webpack --config webpack.config.js ) & pids+=($!) + ( cd conf && npx webpack ) & pids+=($!) for pid in "${pids[@]}"; do wait "$pid" || exit 1; done # to update node-forge library, which is missing the non-minified version in dist, we have to build it manually From a4f456e477df28cb5d09e9077cbe028af5be8e49 Mon Sep 17 00:00:00 2001 From: martgil Date: Fri, 29 Sep 2023 19:39:15 +0800 Subject: [PATCH 14/19] pr reviews: code simplification --- extension/js/common/platform/xss.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/extension/js/common/platform/xss.ts b/extension/js/common/platform/xss.ts index 9be6f4d20f2..6b2cbe32441 100644 --- a/extension/js/common/platform/xss.ts +++ b/extension/js/common/platform/xss.ts @@ -1,7 +1,5 @@ /* ©️ 2016 - present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com */ -/// - 'use strict'; import * as DOMPurify from 'dompurify'; @@ -233,11 +231,7 @@ export class Xss { }; public static stripEmojis = (str: string) => { - if (Intl !== undefined && typeof Intl.Segmenter === 'function') { - str = [...new Intl.Segmenter().segment(str)].map(x => x?.segment).join(''); - } else { - str = str.replace(Xss.EMOJI_REGEX, ''); - } + str = str.replace(Xss.EMOJI_REGEX, ''); return this.escape(str); }; From 75250dfdcfa2dea57d11f4e1b1f892ffded69800 Mon Sep 17 00:00:00 2001 From: martgil Date: Sat, 30 Sep 2023 19:55:06 +0800 Subject: [PATCH 15/19] pr reviews: better test checking --- .../elements/compose-modules/compose-input-module.ts | 2 +- .../formatters/encrypted-mail-msg-formatter.ts | 10 ++++++++++ .../mock/google/strategies/send-message-strategy.ts | 2 ++ test/source/tests/compose.ts | 8 +++++++- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-input-module.ts b/extension/chrome/elements/compose-modules/compose-input-module.ts index d72b495e7c8..245692f69e3 100644 --- a/extension/chrome/elements/compose-modules/compose-input-module.ts +++ b/extension/chrome/elements/compose-modules/compose-input-module.ts @@ -243,7 +243,7 @@ export class ComposeInputModule extends ViewModule { const result: ParsedRecipients = { to: [], cc: [], bcc: [] }; for (const recipient of recipients) { /* eslint-disable @typescript-eslint/no-non-null-assertion */ - const recipientObj = { email: recipient.email, name: Xss.stripEmojis(recipient.name || '') }; + const recipientObj = { email: recipient.email, name: recipient.name || '' }; switch (recipient.sendingType) { case 'to': result.to!.push(recipientObj); diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index e2e9653a8dd..60373e3ee71 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -96,6 +96,16 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { // - flowcrypt.com/shared-tenant-fes (consumers and customers without on-prem setup), or // - fes.customer-domain.com (enterprise customers with on-prem setup) // It will be served to recipient through web + const { to, cc, bcc } = newMsg.recipients; + const updateContactName = (recipient: { email: string; name: string }) => { + return { + ...recipient, + name: Xss.stripEmojis(recipient.name), + }; + }; + newMsg.recipients.to = to?.map(updateContactName); + newMsg.recipients.cc = cc?.map(updateContactName); + newMsg.recipients.bcc = bcc?.map(updateContactName); const uploadedMessageData = await this.prepareAndUploadPwdEncryptedMsg(newMsg, signingKey); // pwdRecipients that have their personal link const individualPwdRecipients = Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).filter(email => !pubkeys.some(p => p.email === email)); diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index 38fe48d213c..f06ad3348f9 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -472,6 +472,8 @@ export class TestBySubjectStrategyContext { this.strategy = new SaveMessageInStorageStrategy(); } else if (subject.includes('Test Sending Message With Attachment Which Contains Emoji in Filename')) { this.strategy = new SaveMessageInStorageStrategy(); + } else if (subject.includes('Test Sending Message With Recipient Name Contains Emoji')) { + this.strategy = new SaveMessageInStorageStrategy(); } else if (subject.includes('Re: FROM: flowcrypt.compatibility@gmail.com, TO: flowcrypt.compatibility@gmail.com + vladimir@flowcrypt.com')) { this.strategy = new NoopTestStrategy(); } else { diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index dbfd2a1be6a..3596a65cd92 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -44,6 +44,8 @@ import { import { Buf } from '../core/buf'; import { flowcryptCompatibilityAliasList, flowcryptCompatibilityPrimarySignature } from '../mock/google/google-endpoints'; import { standardSubDomainFesClientConfiguration } from '../mock/fes/customer-url-fes-endpoints'; +import { AddressObject } from 'mailparser'; +import Parse from '../util/parse'; export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: TestWithBrowser) => { if (testVariant !== 'CONSUMER-LIVE-GMAIL') { @@ -75,11 +77,15 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te }); const recipientEmail = 'User ⭐ Name '; const msgPwd = 'super hard password for the message'; - const subject = 'Strip emojis in display name format'; + const subject = 'Test Sending Message With Recipient Name Contains Emoji'; const composePage = await ComposePageRecipe.openStandalone(t, browser, 'compatibility'); await ComposePageRecipe.selectFromOption(composePage, acct); await ComposePageRecipe.fillMsg(composePage, { to: recipientEmail }, subject); await ComposePageRecipe.sendAndClose(composePage, { password: msgPwd }); + const sentMsg = (await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject)[0]; + const rawMessage = await Parse.convertBase64ToMimeMsg(sentMsg.raw!); + const toHeader = rawMessage.headers.get('to') as AddressObject; + expect(!toHeader.text.includes('⭐')).to.equal(true); }) ); From c600479fec3aa0768a0639ed9f2513a05dfa023c Mon Sep 17 00:00:00 2001 From: martgil Date: Mon, 2 Oct 2023 17:14:51 +0800 Subject: [PATCH 16/19] pr reviews: code and test simplification --- .../compose-modules/compose-recipients-module.ts | 2 +- .../formatters/encrypted-mail-msg-formatter.ts | 10 ---------- .../js/common/api/account-servers/external-service.ts | 7 ++++--- extension/js/common/platform/xss.ts | 3 +-- test/source/mock/fes/shared-tenant-fes-endpoints.ts | 3 +++ test/source/tests/compose.ts | 9 ++------- 6 files changed, 11 insertions(+), 23 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-recipients-module.ts b/extension/chrome/elements/compose-modules/compose-recipients-module.ts index 934e60626b5..0e51015d706 100644 --- a/extension/chrome/elements/compose-modules/compose-recipients-module.ts +++ b/extension/chrome/elements/compose-modules/compose-recipients-module.ts @@ -905,7 +905,7 @@ export class ComposeRecipientsModule extends ViewModule { const recipientId = this.generateRecipientId(); const recipientsHtml = `` + - `${Xss.stripEmojis(name || '')}` + + `${name || ''}` + `${Xss.escape(email || invalid || '')} ${Ui.spinner('green')}`; Xss.sanitizeAppend(container.find('.recipients'), recipientsHtml); const element = document.getElementById(recipientId); diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index 60373e3ee71..e2e9653a8dd 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -96,16 +96,6 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { // - flowcrypt.com/shared-tenant-fes (consumers and customers without on-prem setup), or // - fes.customer-domain.com (enterprise customers with on-prem setup) // It will be served to recipient through web - const { to, cc, bcc } = newMsg.recipients; - const updateContactName = (recipient: { email: string; name: string }) => { - return { - ...recipient, - name: Xss.stripEmojis(recipient.name), - }; - }; - newMsg.recipients.to = to?.map(updateContactName); - newMsg.recipients.cc = cc?.map(updateContactName); - newMsg.recipients.bcc = bcc?.map(updateContactName); const uploadedMessageData = await this.prepareAndUploadPwdEncryptedMsg(newMsg, signingKey); // pwdRecipients that have their personal link const individualPwdRecipients = Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).filter(email => !pubkeys.some(p => p.email === email)); diff --git a/extension/js/common/api/account-servers/external-service.ts b/extension/js/common/api/account-servers/external-service.ts index 11892712030..e9dc1537623 100644 --- a/extension/js/common/api/account-servers/external-service.ts +++ b/extension/js/common/api/account-servers/external-service.ts @@ -14,6 +14,7 @@ import { ClientConfigurationError, ClientConfigurationJson } from '../../client- import { InMemoryStore } from '../../platform/store/in-memory-store.js'; import { GoogleOAuth } from '../authentication/google/google-oauth.js'; import { AuthenticationConfiguration } from '../../authentication-configuration.js'; +import { Xss } from '../../platform/xss.js'; // todo - decide which tags to use type EventTag = 'compose' | 'decrypt' | 'setup' | 'settings' | 'import-pub' | 'import-prv'; @@ -143,9 +144,9 @@ export class ExternalService extends Api { JSON.stringify({ associateReplyToken, from, - to: (recipients.to || []).map(Str.formatEmailWithOptionalName), - cc: (recipients.cc || []).map(Str.formatEmailWithOptionalName), - bcc: (recipients.bcc || []).map(Str.formatEmailWithOptionalName), + to: (recipients.to || []).map(Str.formatEmailWithOptionalName).map(Xss.stripEmojis), + cc: (recipients.cc || []).map(Str.formatEmailWithOptionalName).map(Xss.stripEmojis), + bcc: (recipients.bcc || []).map(Str.formatEmailWithOptionalName).map(Xss.stripEmojis), }) ), }); diff --git a/extension/js/common/platform/xss.ts b/extension/js/common/platform/xss.ts index 6b2cbe32441..7d0240e1e10 100644 --- a/extension/js/common/platform/xss.ts +++ b/extension/js/common/platform/xss.ts @@ -231,8 +231,7 @@ export class Xss { }; public static stripEmojis = (str: string) => { - str = str.replace(Xss.EMOJI_REGEX, ''); - return this.escape(str); + return str.replace(Xss.EMOJI_REGEX, ''); }; public static htmlUnescape = (str: string) => { diff --git a/test/source/mock/fes/shared-tenant-fes-endpoints.ts b/test/source/mock/fes/shared-tenant-fes-endpoints.ts index e0726f6ce31..35e7b5cdffa 100644 --- a/test/source/mock/fes/shared-tenant-fes-endpoints.ts +++ b/test/source/mock/fes/shared-tenant-fes-endpoints.ts @@ -142,6 +142,9 @@ export const getMockSharedTenantFesEndpoints = (config: FesConfig | undefined): if (req.method === 'POST' && typeof body === 'string') { expect(body).to.contain('-----BEGIN PGP MESSAGE-----'); expect(body).to.contain('"associateReplyToken":"mock-fes-reply-token"'); + if (body.includes('NameWithEmoji')) { + expect(body).to.not.include('⭐'); + } const response = { // this url is required for pubkey encrypted message url: `https://flowcrypt.com/shared-tenant-fes/message/6da5ea3c-d2d6-4714-b15e-f29c805e5c6a`, diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index 3596a65cd92..12617388fda 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -44,8 +44,6 @@ import { import { Buf } from '../core/buf'; import { flowcryptCompatibilityAliasList, flowcryptCompatibilityPrimarySignature } from '../mock/google/google-endpoints'; import { standardSubDomainFesClientConfiguration } from '../mock/fes/customer-url-fes-endpoints'; -import { AddressObject } from 'mailparser'; -import Parse from '../util/parse'; export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: TestWithBrowser) => { if (testVariant !== 'CONSUMER-LIVE-GMAIL') { @@ -75,17 +73,14 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te await BrowserRecipe.setupCommonAcctWithAttester(t, browser, 'compatibility', { google: { acctAliases: flowcryptCompatibilityAliasList }, }); - const recipientEmail = 'User ⭐ Name '; + const recipientEmail = 'NameWithEmoji ⭐ Test '; const msgPwd = 'super hard password for the message'; const subject = 'Test Sending Message With Recipient Name Contains Emoji'; const composePage = await ComposePageRecipe.openStandalone(t, browser, 'compatibility'); await ComposePageRecipe.selectFromOption(composePage, acct); await ComposePageRecipe.fillMsg(composePage, { to: recipientEmail }, subject); await ComposePageRecipe.sendAndClose(composePage, { password: msgPwd }); - const sentMsg = (await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject)[0]; - const rawMessage = await Parse.convertBase64ToMimeMsg(sentMsg.raw!); - const toHeader = rawMessage.headers.get('to') as AddressObject; - expect(!toHeader.text.includes('⭐')).to.equal(true); + // The actualt test for this is present in '/shared-tenant-fes/api/v1/message' of shared-tenant-fes mock endpoint. }) ); From 35bd5fed2d911cd0cfa1d43de51c23e1076e4d80 Mon Sep 17 00:00:00 2001 From: martgil Date: Mon, 2 Oct 2023 18:45:13 +0800 Subject: [PATCH 17/19] pr reviews: revert other changes --- .../compose-modules/compose-recipients-module.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-recipients-module.ts b/extension/chrome/elements/compose-modules/compose-recipients-module.ts index 0e51015d706..481fb9975cf 100644 --- a/extension/chrome/elements/compose-modules/compose-recipients-module.ts +++ b/extension/chrome/elements/compose-modules/compose-recipients-module.ts @@ -346,7 +346,7 @@ export class ComposeRecipientsModule extends ViewModule { const recipient = orderedRecipients[processed]; const escapedTitle = Xss.escape(recipient.element.getAttribute('title') || ''); const nameOrEmail = recipient.name || recipient.email || recipient.invalid || ''; - const emailHtml = ``; + const emailHtml = ``; $(emailHtml).insertBefore(rest); // xss-escaped processed++; } @@ -808,7 +808,7 @@ export class ComposeRecipientsModule extends ViewModule { } displayEmail = '
' + Xss.escape(displayEmail) + '
'; if (contact.name) { - ulHtml += '
' + Xss.stripEmojis(contact.name) + displayEmail + '
'; + ulHtml += '
' + Xss.escape(contact.name) + displayEmail + '
'; } else { ulHtml += displayEmail; } @@ -905,7 +905,7 @@ export class ComposeRecipientsModule extends ViewModule { const recipientId = this.generateRecipientId(); const recipientsHtml = `` + - `${name || ''}` + + `${Xss.escape(name || '')}` + `${Xss.escape(email || invalid || '')} ${Ui.spinner('green')}`; Xss.sanitizeAppend(container.find('.recipients'), recipientsHtml); const element = document.getElementById(recipientId); @@ -1056,7 +1056,7 @@ export class ComposeRecipientsModule extends ViewModule { } else if (info && info.sortedPubkeys.length) { if (info.info.name) { recipient.name = info.info.name; - $(el).find('.recipient-name').text(Xss.stripEmojis(info.info.name)); + $(el).find('.recipient-name').text(Xss.escape(info.info.name)); } // New logic: // 1. Keys are sorted in a special way. @@ -1097,7 +1097,7 @@ export class ComposeRecipientsModule extends ViewModule { $(el).addClass('no_pgp'); if (info?.info.name) { recipient.name = info.info.name; - $(el).find('.recipient-name').text(Xss.stripEmojis(info.info.name)); + $(el).find('.recipient-name').text(Xss.escape(info.info.name)); } Xss.sanitizePrepend(el, ''); $(el).attr('title', 'Could not verify their encryption setup. You can encrypt the message with a password below. Alternatively, add their pubkey.'); From 40106cf6df18b0ccdb0a80ba37495ad8ba7613fa Mon Sep 17 00:00:00 2001 From: martgil Date: Tue, 3 Oct 2023 11:54:29 +0800 Subject: [PATCH 18/19] revert unrelated change --- .../elements/compose-modules/compose-input-module.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-input-module.ts b/extension/chrome/elements/compose-modules/compose-input-module.ts index 245692f69e3..c989694e660 100644 --- a/extension/chrome/elements/compose-modules/compose-input-module.ts +++ b/extension/chrome/elements/compose-modules/compose-input-module.ts @@ -243,16 +243,15 @@ export class ComposeInputModule extends ViewModule { const result: ParsedRecipients = { to: [], cc: [], bcc: [] }; for (const recipient of recipients) { /* eslint-disable @typescript-eslint/no-non-null-assertion */ - const recipientObj = { email: recipient.email, name: recipient.name || '' }; switch (recipient.sendingType) { case 'to': - result.to!.push(recipientObj); + result.to!.push({ email: recipient.email, name: recipient.name }); break; case 'cc': - result.cc!.push(recipientObj); + result.cc!.push({ email: recipient.email, name: recipient.name }); break; case 'bcc': - result.bcc!.push(recipientObj); + result.bcc!.push({ email: recipient.email, name: recipient.name }); break; } /* eslint-enable @typescript-eslint/no-non-null-assertion */ From 31106ad3e91725319b1e9885927c05943f12b0fa Mon Sep 17 00:00:00 2001 From: martgil Date: Tue, 3 Oct 2023 14:33:56 +0800 Subject: [PATCH 19/19] cleanup --- test/source/mock/google/strategies/send-message-strategy.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index f06ad3348f9..38fe48d213c 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -472,8 +472,6 @@ export class TestBySubjectStrategyContext { this.strategy = new SaveMessageInStorageStrategy(); } else if (subject.includes('Test Sending Message With Attachment Which Contains Emoji in Filename')) { this.strategy = new SaveMessageInStorageStrategy(); - } else if (subject.includes('Test Sending Message With Recipient Name Contains Emoji')) { - this.strategy = new SaveMessageInStorageStrategy(); } else if (subject.includes('Re: FROM: flowcrypt.compatibility@gmail.com, TO: flowcrypt.compatibility@gmail.com + vladimir@flowcrypt.com')) { this.strategy = new NoopTestStrategy(); } else {