Correct flags scanning for non-BMP characters

graphemecluster · graphemecluster · commit e67692acb3fd · 2024-04-27T15:12:43.000+08:00
diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts
@@ -2485,27 +2485,28 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
                 pos++;
                 let regExpFlags = RegularExpressionFlags.None;
                 while (pos < end) {
-                    const ch = text.charCodeAt(pos);
+                    const ch = codePointAt(text, pos);
                     if (!isIdentifierPart(ch, languageVersion)) {
                         break;
                     }
+                    const size = charSize(ch);
                     if (reportErrors) {
-                        const flag = characterToRegularExpressionFlag(String.fromCharCode(ch));
+                        const flag = characterToRegularExpressionFlag(utf16EncodeAsString(ch));
                         if (flag === undefined) {
-                            error(Diagnostics.Unknown_regular_expression_flag, pos, 1);
+                            error(Diagnostics.Unknown_regular_expression_flag, pos, size);
                         }
                         else if (regExpFlags & flag) {
-                            error(Diagnostics.Duplicate_regular_expression_flag, pos, 1);
+                            error(Diagnostics.Duplicate_regular_expression_flag, pos, size);
                         }
                         else if (((regExpFlags | flag) & RegularExpressionFlags.UnicodeMode) === RegularExpressionFlags.UnicodeMode) {
-                            error(Diagnostics.The_Unicode_u_flag_and_the_Unicode_Sets_v_flag_cannot_be_set_simultaneously, pos, 1);
+                            error(Diagnostics.The_Unicode_u_flag_and_the_Unicode_Sets_v_flag_cannot_be_set_simultaneously, pos, size);
                         }
                         else {
                             regExpFlags |= flag;
-                            checkRegularExpressionFlagAvailable(flag);
+                            checkRegularExpressionFlagAvailability(flag);
                         }
                     }
-                    pos++;
+                    pos += size;
                 }
                 if (reportErrors) {
                     scanRange(startOfRegExpBody, endOfRegExpBody - startOfRegExpBody, () => {
@@ -2752,25 +2753,26 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
 
             function scanPatternModifiers(currFlags: RegularExpressionFlags): RegularExpressionFlags {
                 while (pos < end) {
-                    const ch = text.charCodeAt(pos);
+                    const ch = codePointAt(text, pos);
                     if (!isIdentifierPart(ch, languageVersion)) {
                         break;
                     }
-                    const flag = characterToRegularExpressionFlag(String.fromCharCode(ch));
+                    const size = charSize(ch);
+                    const flag = characterToRegularExpressionFlag(utf16EncodeAsString(ch));
                     if (flag === undefined) {
-                        error(Diagnostics.Unknown_regular_expression_flag, pos, 1);
+                        error(Diagnostics.Unknown_regular_expression_flag, pos, size);
                     }
                     else if (currFlags & flag) {
-                        error(Diagnostics.Duplicate_regular_expression_flag, pos, 1);
+                        error(Diagnostics.Duplicate_regular_expression_flag, pos, size);
                     }
                     else if (!(flag & RegularExpressionFlags.Modifiers)) {
-                        error(Diagnostics.This_regular_expression_flag_cannot_be_toggled_within_a_subpattern, pos, 1);
+                        error(Diagnostics.This_regular_expression_flag_cannot_be_toggled_within_a_subpattern, pos, size);
                     }
                     else {
                         currFlags |= flag;
-                        checkRegularExpressionFlagAvailable(flag);
+                        checkRegularExpressionFlagAvailability(flag);
                     }
-                    pos++;
+                    pos += size;
                 }
                 return currFlags;
             }
@@ -3470,7 +3472,7 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
             });
         }
 
-        function checkRegularExpressionFlagAvailable(flag: RegularExpressionFlags) {
+        function checkRegularExpressionFlagAvailability(flag: RegularExpressionFlags) {
             const availableFrom = regExpFlagToFirstAvailableLanguageVersion.get(flag) as ScriptTarget | undefined;
             if (availableFrom && languageVersion < availableFrom) {
                 error(Diagnostics.This_regular_expression_flag_is_only_available_when_targeting_0_or_later, pos, 1, getNameOfScriptTarget(availableFrom));
diff --git a/tests/baselines/reference/regularExpressionWithNonBMPFlags.errors.txt b/tests/baselines/reference/regularExpressionWithNonBMPFlags.errors.txt
@@ -0,0 +1,23 @@
+regularExpressionWithNonBMPFlags.ts(1,23): error TS1499: Unknown regular expression flag.
+regularExpressionWithNonBMPFlags.ts(1,25): error TS1499: Unknown regular expression flag.
+regularExpressionWithNonBMPFlags.ts(1,28): error TS1499: Unknown regular expression flag.
+regularExpressionWithNonBMPFlags.ts(1,41): error TS1499: Unknown regular expression flag.
+regularExpressionWithNonBMPFlags.ts(1,43): error TS1499: Unknown regular expression flag.
+regularExpressionWithNonBMPFlags.ts(1,45): error TS1499: Unknown regular expression flag.
+
+
+==== regularExpressionWithNonBMPFlags.ts (6 errors) ====
+    const 𝘳𝘦𝘨𝘦𝘹 = /(?𝘴𝘪-𝘮:^𝘧𝘰𝘰.)/𝘨𝘮𝘶;
+                          ~~
+!!! error TS1499: Unknown regular expression flag.
+                            ~~
+!!! error TS1499: Unknown regular expression flag.
+                               ~~
+!!! error TS1499: Unknown regular expression flag.
+                                            ~~
+!!! error TS1499: Unknown regular expression flag.
+                                              ~~
+!!! error TS1499: Unknown regular expression flag.
+                                                ~~
+!!! error TS1499: Unknown regular expression flag.
+    
diff --git a/tests/baselines/reference/regularExpressionWithNonBMPFlags.js b/tests/baselines/reference/regularExpressionWithNonBMPFlags.js
@@ -0,0 +1,8 @@
+//// [tests/cases/compiler/regularExpressionWithNonBMPFlags.ts] ////
+
+//// [regularExpressionWithNonBMPFlags.ts]
+const 𝘳𝘦𝘨𝘦𝘹 = /(?𝘴𝘪-𝘮:^𝘧𝘰𝘰.)/𝘨𝘮𝘶;
+
+
+//// [regularExpressionWithNonBMPFlags.js]
+const 𝘳𝘦𝘨𝘦𝘹 = /(?𝘴𝘪-𝘮:^𝘧𝘰𝘰.)/𝘨𝘮𝘶;
diff --git a/tests/baselines/reference/regularExpressionWithNonBMPFlags.symbols b/tests/baselines/reference/regularExpressionWithNonBMPFlags.symbols
@@ -0,0 +1,6 @@
+//// [tests/cases/compiler/regularExpressionWithNonBMPFlags.ts] ////
+
+=== regularExpressionWithNonBMPFlags.ts ===
+const 𝘳𝘦𝘨𝘦𝘹 = /(?𝘴𝘪-𝘮:^𝘧𝘰𝘰.)/𝘨𝘮𝘶;
+>𝘳𝘦𝘨𝘦𝘹 : Symbol(𝘳𝘦𝘨𝘦𝘹, Decl(regularExpressionWithNonBMPFlags.ts, 0, 5))
+
diff --git a/tests/baselines/reference/regularExpressionWithNonBMPFlags.types b/tests/baselines/reference/regularExpressionWithNonBMPFlags.types
@@ -0,0 +1,9 @@
+//// [tests/cases/compiler/regularExpressionWithNonBMPFlags.ts] ////
+
+=== regularExpressionWithNonBMPFlags.ts ===
+const 𝘳𝘦𝘨𝘦𝘹 = /(?𝘴𝘪-𝘮:^𝘧𝘰𝘰.)/𝘨𝘮𝘶;
+>𝘳𝘦𝘨𝘦𝘹 : RegExp
+>           : ^^^^^^
+>/(?𝘴𝘪-𝘮:^𝘧𝘰𝘰.)/𝘨𝘮𝘶 : RegExp
+>                            : ^^^^^^
+
diff --git a/tests/cases/compiler/regularExpressionWithNonBMPFlags.ts b/tests/cases/compiler/regularExpressionWithNonBMPFlags.ts
@@ -0,0 +1,3 @@
+// @target: esnext
+
+const 𝘳𝘦𝘨𝘦𝘹 = /(?𝘴𝘪-𝘮:^𝘧𝘰𝘰.)/𝘨𝘮𝘶;

Original file line number	Diff line number	Diff line change
`@@ -2485,27 +2485,28 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean`
`2485`	`2485`	`pos++;`
`2486`	`2486`	`let regExpFlags = RegularExpressionFlags.None;`
`2487`	`2487`	`while (pos < end) {`
`2488`		`- const ch = text.charCodeAt(pos);`
	`2488`	`+ const ch = codePointAt(text, pos);`
`2489`	`2489`	`if (!isIdentifierPart(ch, languageVersion)) {`
`2490`	`2490`	`break;`
`2491`	`2491`	`}`
	`2492`	`+ const size = charSize(ch);`
`2492`	`2493`	`if (reportErrors) {`
`2493`		`- const flag = characterToRegularExpressionFlag(String.fromCharCode(ch));`
	`2494`	`+ const flag = characterToRegularExpressionFlag(utf16EncodeAsString(ch));`
`2494`	`2495`	`if (flag === undefined) {`
`2495`		`- error(Diagnostics.Unknown_regular_expression_flag, pos, 1);`
	`2496`	`+ error(Diagnostics.Unknown_regular_expression_flag, pos, size);`
`2496`	`2497`	`}`
`2497`	`2498`	`else if (regExpFlags & flag) {`
`2498`		`- error(Diagnostics.Duplicate_regular_expression_flag, pos, 1);`
	`2499`	`+ error(Diagnostics.Duplicate_regular_expression_flag, pos, size);`
`2499`	`2500`	`}`
`2500`	`2501`	`else if (((regExpFlags \| flag) & RegularExpressionFlags.UnicodeMode) === RegularExpressionFlags.UnicodeMode) {`
`2501`		`- error(Diagnostics.The_Unicode_u_flag_and_the_Unicode_Sets_v_flag_cannot_be_set_simultaneously, pos, 1);`
	`2502`	`+ error(Diagnostics.The_Unicode_u_flag_and_the_Unicode_Sets_v_flag_cannot_be_set_simultaneously, pos, size);`
`2502`	`2503`	`}`
`2503`	`2504`	`else {`
`2504`	`2505`	`regExpFlags \|= flag;`
`2505`		`- checkRegularExpressionFlagAvailable(flag);`
	`2506`	`+ checkRegularExpressionFlagAvailability(flag);`
`2506`	`2507`	`}`
`2507`	`2508`	`}`
`2508`		`- pos++;`
	`2509`	`+ pos += size;`
`2509`	`2510`	`}`
`2510`	`2511`	`if (reportErrors) {`
`2511`	`2512`	`scanRange(startOfRegExpBody, endOfRegExpBody - startOfRegExpBody, () => {`
`@@ -2752,25 +2753,26 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean`
`2752`	`2753`
`2753`	`2754`	`function scanPatternModifiers(currFlags: RegularExpressionFlags): RegularExpressionFlags {`
`2754`	`2755`	`while (pos < end) {`
`2755`		`- const ch = text.charCodeAt(pos);`
	`2756`	`+ const ch = codePointAt(text, pos);`
`2756`	`2757`	`if (!isIdentifierPart(ch, languageVersion)) {`
`2757`	`2758`	`break;`
`2758`	`2759`	`}`
`2759`		`- const flag = characterToRegularExpressionFlag(String.fromCharCode(ch));`
	`2760`	`+ const size = charSize(ch);`
	`2761`	`+ const flag = characterToRegularExpressionFlag(utf16EncodeAsString(ch));`
`2760`	`2762`	`if (flag === undefined) {`
`2761`		`- error(Diagnostics.Unknown_regular_expression_flag, pos, 1);`
	`2763`	`+ error(Diagnostics.Unknown_regular_expression_flag, pos, size);`
`2762`	`2764`	`}`
`2763`	`2765`	`else if (currFlags & flag) {`
`2764`		`- error(Diagnostics.Duplicate_regular_expression_flag, pos, 1);`
	`2766`	`+ error(Diagnostics.Duplicate_regular_expression_flag, pos, size);`
`2765`	`2767`	`}`
`2766`	`2768`	`else if (!(flag & RegularExpressionFlags.Modifiers)) {`
`2767`		`- error(Diagnostics.This_regular_expression_flag_cannot_be_toggled_within_a_subpattern, pos, 1);`
	`2769`	`+ error(Diagnostics.This_regular_expression_flag_cannot_be_toggled_within_a_subpattern, pos, size);`
`2768`	`2770`	`}`
`2769`	`2771`	`else {`
`2770`	`2772`	`currFlags \|= flag;`
`2771`		`- checkRegularExpressionFlagAvailable(flag);`
	`2773`	`+ checkRegularExpressionFlagAvailability(flag);`
`2772`	`2774`	`}`
`2773`		`- pos++;`
	`2775`	`+ pos += size;`
`2774`	`2776`	`}`
`2775`	`2777`	`return currFlags;`
`2776`	`2778`	`}`
`@@ -3470,7 +3472,7 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean`
`3470`	`3472`	`});`
`3471`	`3473`	`}`
`3472`	`3474`
`3473`		`- function checkRegularExpressionFlagAvailable(flag: RegularExpressionFlags) {`
	`3475`	`+ function checkRegularExpressionFlagAvailability(flag: RegularExpressionFlags) {`
`3474`	`3476`	`const availableFrom = regExpFlagToFirstAvailableLanguageVersion.get(flag) as ScriptTarget \| undefined;`
`3475`	`3477`	`if (availableFrom && languageVersion < availableFrom) {`
`3476`	`3478`	`error(Diagnostics.This_regular_expression_flag_is_only_available_when_targeting_0_or_later, pos, 1, getNameOfScriptTarget(availableFrom));`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+// @target: esnext`
	`2`	`+`
	`3`	`+const 𝘳𝘦𝘨𝘦𝘹 = /(?𝘴𝘪-𝘮:^𝘧𝘰𝘰.)/𝘨𝘮𝘶;`