refactor: 識別子の処理に関するリファクタリング

src/parser/plugins/validate-keyword.ts

@@ -3,10 +3,34 @@ import { visitNode } from '../visit.js';
 import type * as Ast from '../../node.js';
 
 // 予約語となっている識別子があるかを確認する。
-// - キーワードは字句解析の段階でそれぞれのKeywordトークンとなるため除外
+// - キーワードは字句解析の段階でそれぞれのKeywordトークンとなるが、エスケープシーケンスを含む場合はIdentifierトークンとなるので検証を行う。
 // - 文脈キーワードは識別子に利用できるため除外
 
 const reservedWord = [
+	// 使用中の語
+	'null',
+	'true',
+	'false',
+	'each',
+	'for',
+	'loop',
+	'do',
+	'while',
+	'break',
+	'continue',
+	'match',
+	'case',
+	'default',
+	'if',
+	'elif',
+	'else',
+	'return',
+	'eval',
+	'var',
+	'let',
+	'exists',
+
+	// 使用予定の語
 	'as',
 	'async',
 	'attr',
@@ -52,25 +76,36 @@ const reservedWord = [
 	'new',
 ];
 
-function throwReservedWordError(name: string, loc: Ast.Loc): void {
-	throw new AiScriptSyntaxError(`Reserved word "${name}" cannot be used as variable name.`, loc.start);
+function validateName(name: string, pos: Ast.Pos): void {
+	if (reservedWord.includes(name)) {
+		throwReservedWordError(name, pos);
+	}
+}
+
+function validateTypeName(name: string, pos: Ast.Pos): void {
+	if (name === 'null') {
+		return;
+	}
+	validateName(name, pos);
+}
+
+function throwReservedWordError(name: string, pos: Ast.Pos): never {
+	throw new AiScriptSyntaxError(`Reserved word "${name}" cannot be used as variable name.`, pos);
 }
 
 function validateDest(node: Ast.Node): Ast.Node {
 	return visitNode(node, node => {
 		switch (node.type) {
 			case 'null': {
-				throwReservedWordError(node.type, node.loc);
+				throwReservedWordError(node.type, node.loc.start);
 				break;
 			}
 			case 'bool': {
-				throwReservedWordError(`${node.value}`, node.loc);
+				throwReservedWordError(`${node.value}`, node.loc.start);
 				break;
 			}
 			case 'identifier': {
-				if (reservedWord.includes(node.name)) {
-					throwReservedWordError(node.name, node.loc);
-				}
+				validateName(node.name, node.loc.start);
 				break;
 			}
 		}
@@ -81,9 +116,7 @@ function validateDest(node: Ast.Node): Ast.Node {
 
 function validateTypeParams(node: Ast.Fn | Ast.FnTypeSource): void {
 	for (const typeParam of node.typeParams) {
-		if (reservedWord.includes(typeParam.name)) {
-			throwReservedWordError(typeParam.name, node.loc);
-		}
+		validateTypeName(typeParam.name, node.loc.start);
 	}
 }
 
@@ -97,48 +130,46 @@ function validateNode(node: Ast.Node): Ast.Node {
 		case 'attr':
 		case 'identifier':
 		case 'prop': {
-			if (reservedWord.includes(node.name)) {
-				throwReservedWordError(node.name, node.loc);
-			}
+			validateName(node.name, node.loc.start);
 			break;
 		}
 		case 'meta': {
-			if (node.name != null && reservedWord.includes(node.name)) {
-				throwReservedWordError(node.name, node.loc);
+			if (node.name != null) {
+				validateName(node.name, node.loc.start);
 			}
 			break;
 		}
 		case 'each': {
-			if (node.label != null && reservedWord.includes(node.label)) {
-				throwReservedWordError(node.label, node.loc);
+			if (node.label != null) {
+				validateName(node.label, node.loc.start);
 			}
 			validateDest(node.var);
 			break;
 		}
 		case 'for': {
-			if (node.label != null && reservedWord.includes(node.label)) {
-				throwReservedWordError(node.label, node.loc);
+			if (node.label != null) {
+				validateName(node.label, node.loc.start);
 			}
-			if (node.var != null && reservedWord.includes(node.var)) {
-				throwReservedWordError(node.var, node.loc);
+			if (node.var != null) {
+				validateName(node.var, node.loc.start);
 			}
 			break;
 		}
 		case 'loop': {
-			if (node.label != null && reservedWord.includes(node.label)) {
-				throwReservedWordError(node.label, node.loc);
+			if (node.label != null) {
+				validateName(node.label, node.loc.start);
 			}
 			break;
 		}
 		case 'break': {
-			if (node.label != null && reservedWord.includes(node.label)) {
-				throwReservedWordError(node.label, node.loc);
+			if (node.label != null) {
+				validateName(node.label, node.loc.start);
 			}
 			break;
 		}
 		case 'continue': {
-			if (node.label != null && reservedWord.includes(node.label)) {
-				throwReservedWordError(node.label, node.loc);
+			if (node.label != null) {
+				validateName(node.label, node.loc.start);
 			}
 			break;
 		}
@@ -150,9 +181,7 @@ function validateNode(node: Ast.Node): Ast.Node {
 			break;
 		}
 		case 'namedTypeSource': {
-			if (reservedWord.includes(node.name)) {
-				throwReservedWordError(node.name, node.loc);
-			}
+			validateTypeName(node.name, node.loc.start);
 			break;
 		}
 		case 'fnTypeSource': {

src/parser/scanner.ts

@@ -1,4 +1,5 @@
 import { AiScriptSyntaxError, AiScriptUnexpectedEOFError } from '../error.js';
+import { decodeUnicodeEscapeSequence } from '../utils/characters.js';
 import { CharStream } from './streams/char-stream.js';
 import { TOKEN, TokenKind } from './token.js';
 import { unexpectedTokenError } from './utils.js';
@@ -9,7 +10,9 @@ import type { Token, TokenPosition } from './token.js';
 const spaceChars = [' ', '\t'];
 const lineBreakChars = ['\r', '\n'];
 const digit = /^[0-9]$/;
-const wordChar = /^[A-Za-z0-9_]$/;
+const identifierStart = /^[A-Za-z_]$/u;
+const identifierPart = /^[A-Za-z0-9_]$/u;
+const hexDigit = /^[0-9a-fA-F]$/;
 const exponentIndicatorPattern = /^[eE]$/;
 
 /**
@@ -282,6 +285,11 @@ export class Scanner implements ITokenStream {
 				}
 				case '\\': {
 					this.stream.next();
+					if (!this.stream.eof && (this.stream.char as string) === 'u') {
+						this.stream.prev();
+						const wordToken = this.tryReadWord(hasLeftSpacing);
+						if (wordToken) return wordToken;
+					}
 					return TOKEN(TokenKind.BackSlash, pos, { hasLeftSpacing });
 				}
 				case ']': {
@@ -332,17 +340,29 @@ export class Scanner implements ITokenStream {
 
 	private tryReadWord(hasLeftSpacing: boolean): Token | undefined {
 		// read a word
-		let value = '';
+		if (this.stream.eof) {
+			return;
+		}
 
 		const pos = this.stream.getPos();
 
-		while (!this.stream.eof && wordChar.test(this.stream.char)) {
-			value += this.stream.char;
-			this.stream.next();
-		}
-		if (value.length === 0) {
+		let rawValue = this.tryReadIdentifierStart();
+		if (rawValue === undefined) {
 			return;
 		}
+		while (!(this.stream.eof as boolean)) {
+			const matchedIdentifierPart = this.tryReadIdentifierPart();
+			if (matchedIdentifierPart === undefined) {
+				break;
+			}
+			rawValue += matchedIdentifierPart;
+		}
+
+		const value = decodeUnicodeEscapeSequence(rawValue);
+		if (value !== rawValue) {
+			throw new AiScriptSyntaxError(`Invalid identifier: "${rawValue}"`, pos);
+		}
+
 		// check word kind
 		switch (value) {
 			case 'null': {
@@ -414,6 +434,56 @@ export class Scanner implements ITokenStream {
 		}
 	}
 
+	private tryReadIdentifierStart(): string | undefined {
+		if (this.stream.eof) {
+			return;
+		}
+		if (identifierStart.test(this.stream.char)) {
+			const value = this.stream.char;
+			this.stream.next();
+			return value;
+		}
+		if (this.stream.char === '\\') {
+			this.stream.next();
+			return '\\' + this.readUnicodeEscapeSequence();
+		}
+		return;
+	}
+
+	private tryReadIdentifierPart(): string | undefined {
+		if (this.stream.eof) {
+			return;
+		}
+		const matchedIdentifierStart = this.tryReadIdentifierStart();
+		if (matchedIdentifierStart !== undefined) {
+			return matchedIdentifierStart;
+		}
+		if (identifierPart.test(this.stream.char)) {
+			const value = this.stream.char;
+			this.stream.next();
+			return value;
+		}
+		return;
+	}
+
+	private readUnicodeEscapeSequence(): `u${string}` {
+		if (this.stream.eof || (this.stream.char as string) !== 'u') {
+			throw new AiScriptSyntaxError('character "u" expected', this.stream.getPos());
+		}
+		this.stream.next();
+
+		let code = '';
+		for (let i = 0; i < 4; i++) {
+			if (this.stream.eof || !hexDigit.test(this.stream.char)) {
+				throw new AiScriptSyntaxError('hexadecimal digit expected', this.stream.getPos());
+			}
+			code += this.stream.char;
+			this.stream.next();
+		}
+
+		return `u${code}`;
+	}
+
 	private tryReadDigits(hasLeftSpacing: boolean): Token | undefined {
 		let wholeNumber = '';
 		let fractional = '';