diff --git a/etc/aiscript.api.md b/etc/aiscript.api.md index f0653e2d..bd332994 100644 --- a/etc/aiscript.api.md +++ b/etc/aiscript.api.md @@ -22,9 +22,9 @@ abstract class AiScriptError extends Error { // (undocumented) info?: any; // (undocumented) - loc?: Loc; - // (undocumented) name: string; + // (undocumented) + pos?: Pos; } // @public @@ -34,11 +34,11 @@ class AiScriptIndexOutOfRangeError extends AiScriptRuntimeError { // @public class AiScriptNamespaceError extends AiScriptError { - constructor(message: string, loc: Loc, info?: any); - // (undocumented) - loc: Loc; + constructor(message: string, pos: Pos, info?: any); // (undocumented) name: string; + // (undocumented) + pos: Pos; } // @public @@ -50,20 +50,20 @@ class AiScriptRuntimeError extends AiScriptError { // @public class AiScriptSyntaxError extends AiScriptError { - constructor(message: string, loc: Loc, info?: any); - // (undocumented) - loc: Loc; + constructor(message: string, pos: Pos, info?: any); // (undocumented) name: string; + // (undocumented) + pos: Pos; } // @public class AiScriptTypeError extends AiScriptError { - constructor(message: string, loc: Loc, info?: any); - // (undocumented) - loc: Loc; + constructor(message: string, pos: Pos, info?: any); // (undocumented) name: string; + // (undocumented) + pos: Pos; } // @public @@ -119,6 +119,7 @@ declare namespace Ast { export { isStatement, isExpression, + Pos, Loc, Node_2 as Node, Namespace, @@ -390,10 +391,10 @@ function isString(val: Value): val is VStr; // @public (undocumented) function jsToVal(val: any): Value; -// @public +// @public (undocumented) type Loc = { - line: number; - column: number; + start: Pos; + end: Pos; }; // @public (undocumented) @@ -503,6 +504,12 @@ export type ParserPlugin = (nodes: Ast.Node[]) => Ast.Node[]; // @public (undocumented) export type PluginType = 'validate' | 'transform'; +// @public +type Pos = { + line: number; + column: number; +}; + // @public (undocumented) type Prop = NodeBase & { type: 'prop'; diff --git a/src/error.ts b/src/error.ts index 6d66c0f5..4f1ffb53 100644 --- a/src/error.ts +++ b/src/error.ts @@ -1,11 +1,11 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import type { Loc } from './node.js'; +import type { Pos } from './node.js'; export abstract class AiScriptError extends Error { // name is read by Error.prototype.toString public name = 'AiScript'; public info?: any; - public loc?: Loc; + public pos?: Pos; constructor(message: string, info?: any) { super(message); @@ -34,8 +34,8 @@ export class NonAiScriptError extends AiScriptError { */ export class AiScriptSyntaxError extends AiScriptError { public name = 'Syntax'; - constructor(message: string, public loc: Loc, info?: any) { - super(`${message} (Line ${loc.line}, Column ${loc.column})`, info); + constructor(message: string, public pos: Pos, info?: any) { + super(`${message} (Line ${pos.line}, Column ${pos.column})`, info); } } /** @@ -43,8 +43,8 @@ export class AiScriptSyntaxError extends AiScriptError { */ export class AiScriptTypeError extends AiScriptError { public name = 'Type'; - constructor(message: string, public loc: Loc, info?: any) { - super(`${message} (Line ${loc.line}, Column ${loc.column})`, info); + constructor(message: string, public pos: Pos, info?: any) { + super(`${message} (Line ${pos.line}, Column ${pos.column})`, info); } } @@ -53,8 +53,8 @@ export class AiScriptTypeError extends AiScriptError { */ export class AiScriptNamespaceError extends AiScriptError { public name = 'Namespace'; - constructor(message: string, public loc: Loc, info?: any) { - super(`${message} (Line ${loc.line}, Column ${loc.column})`, info); + constructor(message: string, public pos: Pos, info?: any) { + super(`${message} (Line ${pos.line}, Column ${pos.column})`, info); } } diff --git a/src/interpreter/index.ts b/src/interpreter/index.ts index 38c71545..1899731e 100644 --- a/src/interpreter/index.ts +++ b/src/interpreter/index.ts @@ -191,7 +191,7 @@ export class Interpreter { switch (node.type) { case 'def': { if (node.mut) { - throw new AiScriptNamespaceError('No "var" in namespace declaration: ' + node.name, node.loc); + throw new AiScriptNamespaceError('No "var" in namespace declaration: ' + node.name, node.loc.start); } const variable: Variable = { @@ -211,7 +211,7 @@ export class Interpreter { // exhaustiveness check const n: never = node; const nd = n as Ast.Node; - throw new AiScriptNamespaceError('invalid ns member type: ' + nd.type, nd.loc); + throw new AiScriptNamespaceError('invalid ns member type: ' + nd.type, nd.loc.start); } } } @@ -246,11 +246,11 @@ export class Interpreter { @autobind private _eval(node: Ast.Node, scope: Scope): Promise { return this.__eval(node, scope).catch(e => { - if (e.loc) throw e; + if (e.pos) throw e; else { const e2 = (e instanceof AiScriptError) ? e : new NonAiScriptError(e); - e2.loc = node.loc; - e2.message = `${e2.message} (Line ${node.loc.line}, Column ${node.loc.column})`; + e2.pos = node.loc.start; + e2.message = `${e2.message} (Line ${e2.pos.line}, Column ${e2.pos.column})`; throw e2; } }); diff --git a/src/node.ts b/src/node.ts index 43b556fd..ebc4ad62 100644 --- a/src/node.ts +++ b/src/node.ts @@ -2,11 +2,16 @@ * ASTノード */ -export type Loc = { +export type Pos = { line: number; column: number; }; +export type Loc = { + start: Pos; + end: Pos; +}; + export type Node = Namespace | Meta | Statement | Expression | TypeSource | Attribute; type NodeBase = { diff --git a/src/parser/plugins/validate-keyword.ts b/src/parser/plugins/validate-keyword.ts index 97b1a3cf..d44da8f9 100644 --- a/src/parser/plugins/validate-keyword.ts +++ b/src/parser/plugins/validate-keyword.ts @@ -53,7 +53,7 @@ const reservedWord = [ ]; function throwReservedWordError(name: string, loc: Ast.Loc): void { - throw new AiScriptSyntaxError(`Reserved word "${name}" cannot be used as variable name.`, loc); + throw new AiScriptSyntaxError(`Reserved word "${name}" cannot be used as variable name.`, loc.start); } function validateNode(node: Ast.Node): Ast.Node { diff --git a/src/parser/scanner.ts b/src/parser/scanner.ts index fb2d4c17..4feb5ee7 100644 --- a/src/parser/scanner.ts +++ b/src/parser/scanner.ts @@ -3,7 +3,7 @@ import { CharStream } from './streams/char-stream.js'; import { TOKEN, TokenKind } from './token.js'; import type { ITokenStream } from './streams/token-stream.js'; -import type { Token } from './token.js'; +import type { Token, TokenPosition } from './token.js'; const spaceChars = [' ', '\t']; const lineBreakChars = ['\r', '\n']; @@ -42,6 +42,13 @@ export class Scanner implements ITokenStream { return this.token.kind; } + /** + * カーソル位置にあるトークンの位置情報を取得します。 + */ + public getPos(): TokenPosition { + return this.token.pos; + } + /** * カーソル位置を次のトークンへ進めます。 */ @@ -75,7 +82,7 @@ export class Scanner implements ITokenStream { */ public expect(kind: TokenKind): void { if (this.getKind() !== kind) { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[this.getKind()]}`, this.token.loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[this.getKind()]}`, this.getPos()); } } @@ -105,11 +112,11 @@ export class Scanner implements ITokenStream { } // トークン位置を記憶 - const loc = this.stream.getPos(); + const pos = this.stream.getPos(); if (lineBreakChars.includes(this.stream.char)) { this.stream.next(); - token = TOKEN(TokenKind.NewLine, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.NewLine, pos, { hasLeftSpacing }); return token; } switch (this.stream.char) { @@ -117,9 +124,9 @@ export class Scanner implements ITokenStream { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '=') { this.stream.next(); - token = TOKEN(TokenKind.NotEq, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.NotEq, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Not, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Not, pos, { hasLeftSpacing }); } break; } @@ -134,72 +141,72 @@ export class Scanner implements ITokenStream { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '#') { this.stream.next(); - token = TOKEN(TokenKind.Sharp3, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Sharp3, pos, { hasLeftSpacing }); } } else if (!this.stream.eof && (this.stream.char as string) === '[') { this.stream.next(); - token = TOKEN(TokenKind.OpenSharpBracket, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.OpenSharpBracket, pos, { hasLeftSpacing }); } else { - throw new AiScriptSyntaxError('invalid character: "#"', loc); + throw new AiScriptSyntaxError('invalid character: "#"', pos); } break; } case '%': { this.stream.next(); - token = TOKEN(TokenKind.Percent, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Percent, pos, { hasLeftSpacing }); break; } case '&': { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '&') { this.stream.next(); - token = TOKEN(TokenKind.And2, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.And2, pos, { hasLeftSpacing }); } break; } case '(': { this.stream.next(); - token = TOKEN(TokenKind.OpenParen, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.OpenParen, pos, { hasLeftSpacing }); break; } case ')': { this.stream.next(); - token = TOKEN(TokenKind.CloseParen, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.CloseParen, pos, { hasLeftSpacing }); break; } case '*': { this.stream.next(); - token = TOKEN(TokenKind.Asterisk, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Asterisk, pos, { hasLeftSpacing }); break; } case '+': { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '=') { this.stream.next(); - token = TOKEN(TokenKind.PlusEq, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.PlusEq, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Plus, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Plus, pos, { hasLeftSpacing }); } break; } case ',': { this.stream.next(); - token = TOKEN(TokenKind.Comma, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Comma, pos, { hasLeftSpacing }); break; } case '-': { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '=') { this.stream.next(); - token = TOKEN(TokenKind.MinusEq, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.MinusEq, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Minus, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Minus, pos, { hasLeftSpacing }); } break; } case '.': { this.stream.next(); - token = TOKEN(TokenKind.Dot, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Dot, pos, { hasLeftSpacing }); break; } case '/': { @@ -213,7 +220,7 @@ export class Scanner implements ITokenStream { this.skipCommentLine(); continue; } else { - token = TOKEN(TokenKind.Slash, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Slash, pos, { hasLeftSpacing }); } break; } @@ -221,27 +228,27 @@ export class Scanner implements ITokenStream { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === ':') { this.stream.next(); - token = TOKEN(TokenKind.Colon2, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Colon2, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Colon, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Colon, pos, { hasLeftSpacing }); } break; } case ';': { this.stream.next(); - token = TOKEN(TokenKind.SemiColon, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.SemiColon, pos, { hasLeftSpacing }); break; } case '<': { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '=') { this.stream.next(); - token = TOKEN(TokenKind.LtEq, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.LtEq, pos, { hasLeftSpacing }); } else if (!this.stream.eof && (this.stream.char as string) === ':') { this.stream.next(); - token = TOKEN(TokenKind.Out, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Out, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Lt, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Lt, pos, { hasLeftSpacing }); } break; } @@ -249,12 +256,12 @@ export class Scanner implements ITokenStream { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '=') { this.stream.next(); - token = TOKEN(TokenKind.Eq2, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Eq2, pos, { hasLeftSpacing }); } else if (!this.stream.eof && (this.stream.char as string) === '>') { this.stream.next(); - token = TOKEN(TokenKind.Arrow, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Arrow, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Eq, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Eq, pos, { hasLeftSpacing }); } break; } @@ -262,40 +269,40 @@ export class Scanner implements ITokenStream { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '=') { this.stream.next(); - token = TOKEN(TokenKind.GtEq, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.GtEq, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Gt, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Gt, pos, { hasLeftSpacing }); } break; } case '?': { this.stream.next(); - token = TOKEN(TokenKind.Question, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Question, pos, { hasLeftSpacing }); break; } case '@': { this.stream.next(); - token = TOKEN(TokenKind.At, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.At, pos, { hasLeftSpacing }); break; } case '[': { this.stream.next(); - token = TOKEN(TokenKind.OpenBracket, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.OpenBracket, pos, { hasLeftSpacing }); break; } case '\\': { this.stream.next(); - token = TOKEN(TokenKind.BackSlash, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.BackSlash, pos, { hasLeftSpacing }); break; } case ']': { this.stream.next(); - token = TOKEN(TokenKind.CloseBracket, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.CloseBracket, pos, { hasLeftSpacing }); break; } case '^': { this.stream.next(); - token = TOKEN(TokenKind.Hat, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Hat, pos, { hasLeftSpacing }); break; } case '`': { @@ -304,20 +311,20 @@ export class Scanner implements ITokenStream { } case '{': { this.stream.next(); - token = TOKEN(TokenKind.OpenBrace, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.OpenBrace, pos, { hasLeftSpacing }); break; } case '|': { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '|') { this.stream.next(); - token = TOKEN(TokenKind.Or2, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Or2, pos, { hasLeftSpacing }); } break; } case '}': { this.stream.next(); - token = TOKEN(TokenKind.CloseBrace, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.CloseBrace, pos, { hasLeftSpacing }); break; } } @@ -332,7 +339,7 @@ export class Scanner implements ITokenStream { token = wordToken; break; } - throw new AiScriptSyntaxError(`invalid character: "${this.stream.char}"`, loc); + throw new AiScriptSyntaxError(`invalid character: "${this.stream.char}"`, pos); } break; } @@ -343,7 +350,7 @@ export class Scanner implements ITokenStream { // read a word let value = ''; - const loc = this.stream.getPos(); + const pos = this.stream.getPos(); while (!this.stream.eof && wordChar.test(this.stream.char)) { value += this.stream.char; @@ -355,70 +362,70 @@ export class Scanner implements ITokenStream { // check word kind switch (value) { case 'null': { - return TOKEN(TokenKind.NullKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.NullKeyword, pos, { hasLeftSpacing }); } case 'true': { - return TOKEN(TokenKind.TrueKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.TrueKeyword, pos, { hasLeftSpacing }); } case 'false': { - return TOKEN(TokenKind.FalseKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.FalseKeyword, pos, { hasLeftSpacing }); } case 'each': { - return TOKEN(TokenKind.EachKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.EachKeyword, pos, { hasLeftSpacing }); } case 'for': { - return TOKEN(TokenKind.ForKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.ForKeyword, pos, { hasLeftSpacing }); } case 'loop': { - return TOKEN(TokenKind.LoopKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.LoopKeyword, pos, { hasLeftSpacing }); } case 'do': { - return TOKEN(TokenKind.DoKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.DoKeyword, pos, { hasLeftSpacing }); } case 'while': { - return TOKEN(TokenKind.WhileKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.WhileKeyword, pos, { hasLeftSpacing }); } case 'break': { - return TOKEN(TokenKind.BreakKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.BreakKeyword, pos, { hasLeftSpacing }); } case 'continue': { - return TOKEN(TokenKind.ContinueKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.ContinueKeyword, pos, { hasLeftSpacing }); } case 'match': { - return TOKEN(TokenKind.MatchKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.MatchKeyword, pos, { hasLeftSpacing }); } case 'case': { - return TOKEN(TokenKind.CaseKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.CaseKeyword, pos, { hasLeftSpacing }); } case 'default': { - return TOKEN(TokenKind.DefaultKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.DefaultKeyword, pos, { hasLeftSpacing }); } case 'if': { - return TOKEN(TokenKind.IfKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.IfKeyword, pos, { hasLeftSpacing }); } case 'elif': { - return TOKEN(TokenKind.ElifKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.ElifKeyword, pos, { hasLeftSpacing }); } case 'else': { - return TOKEN(TokenKind.ElseKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.ElseKeyword, pos, { hasLeftSpacing }); } case 'return': { - return TOKEN(TokenKind.ReturnKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.ReturnKeyword, pos, { hasLeftSpacing }); } case 'eval': { - return TOKEN(TokenKind.EvalKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.EvalKeyword, pos, { hasLeftSpacing }); } case 'var': { - return TOKEN(TokenKind.VarKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.VarKeyword, pos, { hasLeftSpacing }); } case 'let': { - return TOKEN(TokenKind.LetKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.LetKeyword, pos, { hasLeftSpacing }); } case 'exists': { - return TOKEN(TokenKind.ExistsKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.ExistsKeyword, pos, { hasLeftSpacing }); } default: { - return TOKEN(TokenKind.Identifier, loc, { hasLeftSpacing, value }); + return TOKEN(TokenKind.Identifier, pos, { hasLeftSpacing, value }); } } } @@ -427,7 +434,7 @@ export class Scanner implements ITokenStream { let wholeNumber = ''; let fractional = ''; - const loc = this.stream.getPos(); + const pos = this.stream.getPos(); while (!this.stream.eof && digit.test(this.stream.char)) { wholeNumber += this.stream.char; @@ -443,7 +450,7 @@ export class Scanner implements ITokenStream { this.stream.next(); } if (fractional.length === 0) { - throw new AiScriptSyntaxError('digit expected', loc); + throw new AiScriptSyntaxError('digit expected', pos); } } let value; @@ -452,7 +459,7 @@ export class Scanner implements ITokenStream { } else { value = wholeNumber; } - return TOKEN(TokenKind.NumberLiteral, loc, { hasLeftSpacing, value }); + return TOKEN(TokenKind.NumberLiteral, pos, { hasLeftSpacing, value }); } private readStringLiteral(hasLeftSpacing: boolean): Token { @@ -460,14 +467,14 @@ export class Scanner implements ITokenStream { const literalMark = this.stream.char; let state: 'string' | 'escape' | 'finish' = 'string'; - const loc = this.stream.getPos(); + const pos = this.stream.getPos(); this.stream.next(); while (state !== 'finish') { switch (state) { case 'string': { if (this.stream.eof) { - throw new AiScriptSyntaxError('unexpected EOF', loc); + throw new AiScriptSyntaxError('unexpected EOF', pos); } if (this.stream.char === '\\') { this.stream.next(); @@ -485,7 +492,7 @@ export class Scanner implements ITokenStream { } case 'escape': { if (this.stream.eof) { - throw new AiScriptSyntaxError('unexpected EOF', loc); + throw new AiScriptSyntaxError('unexpected EOF', pos); } value += this.stream.char; this.stream.next(); @@ -494,7 +501,7 @@ export class Scanner implements ITokenStream { } } } - return TOKEN(TokenKind.StringLiteral, loc, { hasLeftSpacing, value }); + return TOKEN(TokenKind.StringLiteral, pos, { hasLeftSpacing, value }); } private readTemplate(hasLeftSpacing: boolean): Token { @@ -503,8 +510,8 @@ export class Scanner implements ITokenStream { let tokenBuf: Token[] = []; let state: 'string' | 'escape' | 'expr' | 'finish' = 'string'; - const loc = this.stream.getPos(); - let elementLoc = loc; + const pos = this.stream.getPos(); + let elementPos = pos; this.stream.next(); while (state !== 'finish') { @@ -512,7 +519,7 @@ export class Scanner implements ITokenStream { case 'string': { // テンプレートの終了が無いままEOFに達した if (this.stream.eof) { - throw new AiScriptSyntaxError('unexpected EOF', loc); + throw new AiScriptSyntaxError('unexpected EOF', pos); } // エスケープ if (this.stream.char === '\\') { @@ -524,7 +531,7 @@ export class Scanner implements ITokenStream { if (this.stream.char === '`') { this.stream.next(); if (buf.length > 0) { - elements.push(TOKEN(TokenKind.TemplateStringElement, elementLoc, { hasLeftSpacing, value: buf })); + elements.push(TOKEN(TokenKind.TemplateStringElement, elementPos, { hasLeftSpacing, value: buf })); } state = 'finish'; break; @@ -533,11 +540,11 @@ export class Scanner implements ITokenStream { if (this.stream.char === '{') { this.stream.next(); if (buf.length > 0) { - elements.push(TOKEN(TokenKind.TemplateStringElement, elementLoc, { hasLeftSpacing, value: buf })); + elements.push(TOKEN(TokenKind.TemplateStringElement, elementPos, { hasLeftSpacing, value: buf })); buf = ''; } // ここから式エレメントになるので位置を更新 - elementLoc = this.stream.getPos(); + elementPos = this.stream.getPos(); state = 'expr'; break; } @@ -548,7 +555,7 @@ export class Scanner implements ITokenStream { case 'escape': { // エスケープ対象の文字が無いままEOFに達した if (this.stream.eof) { - throw new AiScriptSyntaxError('unexpected EOF', loc); + throw new AiScriptSyntaxError('unexpected EOF', pos); } // 普通の文字として取り込み buf += this.stream.char; @@ -560,7 +567,7 @@ export class Scanner implements ITokenStream { case 'expr': { // 埋め込み式の終端記号が無いままEOFに達した if (this.stream.eof) { - throw new AiScriptSyntaxError('unexpected EOF', loc); + throw new AiScriptSyntaxError('unexpected EOF', pos); } // skip spasing if (spaceChars.includes(this.stream.char)) { @@ -569,12 +576,14 @@ export class Scanner implements ITokenStream { } // 埋め込み式の終了 if ((this.stream.char as string) === '}') { - this.stream.next(); - elements.push(TOKEN(TokenKind.TemplateExprElement, elementLoc, { hasLeftSpacing, children: tokenBuf })); - tokenBuf = []; + elements.push(TOKEN(TokenKind.TemplateExprElement, elementPos, { hasLeftSpacing, children: tokenBuf })); // ここから文字列エレメントになるので位置を更新 - elementLoc = this.stream.getPos(); + elementPos = this.stream.getPos(); + // TemplateExprElementトークンの終了位置をTokenStreamが取得するためのEOFトークンを追加 + tokenBuf.push(TOKEN(TokenKind.EOF, elementPos)); + tokenBuf = []; state = 'string'; + this.stream.next(); break; } const token = this.readToken(); @@ -584,7 +593,7 @@ export class Scanner implements ITokenStream { } } - return TOKEN(TokenKind.Template, loc, { hasLeftSpacing, children: elements }); + return TOKEN(TokenKind.Template, pos, { hasLeftSpacing, children: elements }); } private skipCommentLine(): void { diff --git a/src/parser/streams/token-stream.ts b/src/parser/streams/token-stream.ts index 0ef2fe23..d6d3f68b 100644 --- a/src/parser/streams/token-stream.ts +++ b/src/parser/streams/token-stream.ts @@ -1,6 +1,6 @@ import { AiScriptSyntaxError } from '../../error.js'; import { TOKEN, TokenKind } from '../token.js'; -import type { Token } from '../token.js'; +import type { Token, TokenPosition } from '../token.js'; /** * トークンの読み取りに関するインターフェース @@ -16,6 +16,11 @@ export interface ITokenStream { */ getKind(): TokenKind; + /** + * カーソル位置にあるトークンの位置情報を取得します。 + */ + getPos(): TokenPosition; + /** * カーソル位置を次のトークンへ進めます。 */ @@ -74,6 +79,13 @@ export class TokenStream implements ITokenStream { return this.token.kind; } + /** + * カーソル位置にあるトークンの位置情報を取得します。 + */ + public getPos(): TokenPosition { + return this.token.pos; + } + /** * カーソル位置を次のトークンへ進めます。 */ @@ -101,7 +113,7 @@ export class TokenStream implements ITokenStream { */ public expect(kind: TokenKind): void { if (this.getKind() !== kind) { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[this.getKind()]}`, this.token.loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[this.getKind()]}`, this.getPos()); } } diff --git a/src/parser/syntaxes/common.ts b/src/parser/syntaxes/common.ts index f185a158..faf5cc5e 100644 --- a/src/parser/syntaxes/common.ts +++ b/src/parser/syntaxes/common.ts @@ -60,7 +60,7 @@ export function parseParams(s: ITokenStream): { name: string, argType?: Ast.Node break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } @@ -99,7 +99,7 @@ export function parseBlock(s: ITokenStream): Ast.Node[] { break; } default: { - throw new AiScriptSyntaxError('Multiple statements cannot be placed on a single line.', s.token.loc); + throw new AiScriptSyntaxError('Multiple statements cannot be placed on a single line.', s.getPos()); } } } @@ -126,7 +126,7 @@ export function parseType(s: ITokenStream): Ast.Node { * ``` */ function parseFnType(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.At); s.nextWith(TokenKind.OpenParen); @@ -140,7 +140,7 @@ function parseFnType(s: ITokenStream): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } @@ -153,7 +153,7 @@ function parseFnType(s: ITokenStream): Ast.Node { const resultType = parseType(s); - return NODE('fnTypeSource', { args: params, result: resultType }, loc); + return NODE('fnTypeSource', { args: params, result: resultType }, startPos, s.getPos()); } /** @@ -162,7 +162,7 @@ function parseFnType(s: ITokenStream): Ast.Node { * ``` */ function parseNamedType(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.expect(TokenKind.Identifier); const name = s.token.value!; @@ -176,7 +176,7 @@ function parseNamedType(s: ITokenStream): Ast.Node { s.nextWith(TokenKind.Gt); } - return NODE('namedTypeSource', { name, inner }, loc); + return NODE('namedTypeSource', { name, inner }, startPos, s.getPos()); } //#endregion Type diff --git a/src/parser/syntaxes/expressions.ts b/src/parser/syntaxes/expressions.ts index 439b8fc1..9fd0b893 100644 --- a/src/parser/syntaxes/expressions.ts +++ b/src/parser/syntaxes/expressions.ts @@ -52,7 +52,7 @@ const operators: OpInfo[] = [ ]; function parsePrefix(s: ITokenStream, minBp: number): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); const op = s.getKind(); s.next(); @@ -64,38 +64,40 @@ function parsePrefix(s: ITokenStream, minBp: number): Ast.Node { const expr = parsePratt(s, minBp); + const endPos = expr.loc.end; + switch (op) { case TokenKind.Plus: { // 数値リテラル以外は非サポート if (expr.type === 'num') { - return NODE('num', { value: expr.value }, loc); + return NODE('num', { value: expr.value }, startPos, endPos); } else { - throw new AiScriptSyntaxError('currently, sign is only supported for number literal.', loc); + throw new AiScriptSyntaxError('currently, sign is only supported for number literal.', startPos); } // TODO: 将来的にサポートされる式を拡張 - // return NODE('plus', { expr }, loc); + // return NODE('plus', { expr }, startPos, endPos); } case TokenKind.Minus: { // 数値リテラル以外は非サポート if (expr.type === 'num') { - return NODE('num', { value: -1 * expr.value }, loc); + return NODE('num', { value: -1 * expr.value }, startPos, endPos); } else { - throw new AiScriptSyntaxError('currently, sign is only supported for number literal.', loc); + throw new AiScriptSyntaxError('currently, sign is only supported for number literal.', startPos); } // TODO: 将来的にサポートされる式を拡張 - // return NODE('minus', { expr }, loc); + // return NODE('minus', { expr }, startPos, endPos); } case TokenKind.Not: { - return NODE('not', { expr }, loc); + return NODE('not', { expr }, startPos, endPos); } default: { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[op]}`, loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[op]}`, startPos); } } } function parseInfix(s: ITokenStream, left: Ast.Node, minBp: number): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); const op = s.getKind(); s.next(); @@ -113,62 +115,63 @@ function parseInfix(s: ITokenStream, left: Ast.Node, minBp: number): Ast.Node { return NODE('prop', { target: left, name, - }, loc); + }, startPos, s.getPos()); } else { const right = parsePratt(s, minBp); + const endPos = s.getPos(); switch (op) { case TokenKind.Hat: { - return CALL_NODE('Core:pow', [left, right], loc); + return CALL_NODE('Core:pow', [left, right], startPos, endPos); } case TokenKind.Asterisk: { - return CALL_NODE('Core:mul', [left, right], loc); + return CALL_NODE('Core:mul', [left, right], startPos, endPos); } case TokenKind.Slash: { - return CALL_NODE('Core:div', [left, right], loc); + return CALL_NODE('Core:div', [left, right], startPos, endPos); } case TokenKind.Percent: { - return CALL_NODE('Core:mod', [left, right], loc); + return CALL_NODE('Core:mod', [left, right], startPos, endPos); } case TokenKind.Plus: { - return CALL_NODE('Core:add', [left, right], loc); + return CALL_NODE('Core:add', [left, right], startPos, endPos); } case TokenKind.Minus: { - return CALL_NODE('Core:sub', [left, right], loc); + return CALL_NODE('Core:sub', [left, right], startPos, endPos); } case TokenKind.Lt: { - return CALL_NODE('Core:lt', [left, right], loc); + return CALL_NODE('Core:lt', [left, right], startPos, endPos); } case TokenKind.LtEq: { - return CALL_NODE('Core:lteq', [left, right], loc); + return CALL_NODE('Core:lteq', [left, right], startPos, endPos); } case TokenKind.Gt: { - return CALL_NODE('Core:gt', [left, right], loc); + return CALL_NODE('Core:gt', [left, right], startPos, endPos); } case TokenKind.GtEq: { - return CALL_NODE('Core:gteq', [left, right], loc); + return CALL_NODE('Core:gteq', [left, right], startPos, endPos); } case TokenKind.Eq2: { - return CALL_NODE('Core:eq', [left, right], loc); + return CALL_NODE('Core:eq', [left, right], startPos, endPos); } case TokenKind.NotEq: { - return CALL_NODE('Core:neq', [left, right], loc); + return CALL_NODE('Core:neq', [left, right], startPos, endPos); } case TokenKind.And2: { - return NODE('and', { left, right }, loc); + return NODE('and', { left, right }, startPos, endPos); } case TokenKind.Or2: { - return NODE('or', { left, right }, loc); + return NODE('or', { left, right }, startPos, endPos); } default: { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[op]}`, loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[op]}`, startPos); } } } } function parsePostfix(s: ITokenStream, expr: Ast.Node): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); const op = s.getKind(); switch (op) { @@ -183,16 +186,16 @@ function parsePostfix(s: ITokenStream, expr: Ast.Node): Ast.Node { return NODE('index', { target: expr, index, - }, loc); + }, startPos, s.getPos()); } default: { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[op]}`, loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[op]}`, startPos); } } } function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); switch (s.getKind()) { case TokenKind.IfKeyword: { @@ -220,10 +223,12 @@ function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Node { if (isStatic) break; - for (const element of s.token.children!) { + for (const [i, element] of s.token.children!.entries()) { switch (element.kind) { case TokenKind.TemplateStringElement: { - values.push(NODE('str', { value: element.value! }, element.loc)); + // トークンの終了位置を取得するために先読み + const nextToken = s.token.children![i + 1] ?? s.lookahead(1); + values.push(NODE('str', { value: element.value! }, element.pos, nextToken.pos)); break; } case TokenKind.TemplateExprElement: { @@ -231,40 +236,40 @@ function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Node { const exprStream = new TokenStream(element.children!); const expr = parseExpr(exprStream, false); if (exprStream.getKind() !== TokenKind.EOF) { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[exprStream.token.kind]}`, exprStream.token.loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[exprStream.token.kind]}`, exprStream.token.pos); } values.push(expr); break; } default: { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[element.kind]}`, element.loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[element.kind]}`, element.pos); } } } s.next(); - return NODE('tmpl', { tmpl: values }, loc); + return NODE('tmpl', { tmpl: values }, startPos, s.getPos()); } case TokenKind.StringLiteral: { const value = s.token.value!; s.next(); - return NODE('str', { value }, loc); + return NODE('str', { value }, startPos, s.getPos()); } case TokenKind.NumberLiteral: { // TODO: validate number value const value = Number(s.token.value!); s.next(); - return NODE('num', { value }, loc); + return NODE('num', { value }, startPos, s.getPos()); } case TokenKind.TrueKeyword: case TokenKind.FalseKeyword: { const value = (s.getKind() === TokenKind.TrueKeyword); s.next(); - return NODE('bool', { value }, loc); + return NODE('bool', { value }, startPos, s.getPos()); } case TokenKind.NullKeyword: { s.next(); - return NODE('null', { }, loc); + return NODE('null', {}, startPos, s.getPos()); } case TokenKind.OpenBrace: { return parseObject(s, isStatic); @@ -283,14 +288,14 @@ function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Node { return expr; } } - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[s.getKind()]}`, loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[s.getKind()]}`, startPos); } /** * Call = "(" [Expr *(SEP Expr) [SEP]] ")" */ function parseCall(s: ITokenStream, target: Ast.Node): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); const items: Ast.Node[] = []; s.nextWith(TokenKind.OpenParen); @@ -319,7 +324,7 @@ function parseCall(s: ITokenStream, target: Ast.Node): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } @@ -329,7 +334,7 @@ function parseCall(s: ITokenStream, target: Ast.Node): Ast.Node { return NODE('call', { target, args: items, - }, loc); + }, startPos, s.getPos()); } /** @@ -338,7 +343,7 @@ function parseCall(s: ITokenStream, target: Ast.Node): Ast.Node { * ``` */ function parseIf(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.IfKeyword); const cond = parseExpr(s, false); @@ -365,7 +370,7 @@ function parseIf(s: ITokenStream): Ast.Node { _else = parseBlockOrStatement(s); } - return NODE('if', { cond, then, elseif, else: _else }, loc); + return NODE('if', { cond, then, elseif, else: _else }, startPos, s.getPos()); } /** @@ -374,7 +379,7 @@ function parseIf(s: ITokenStream): Ast.Node { * ``` */ function parseFnExpr(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.At); @@ -388,7 +393,7 @@ function parseFnExpr(s: ITokenStream): Ast.Node { const body = parseBlock(s); - return NODE('fn', { args: params, retType: type, children: body }, loc); + return NODE('fn', { args: params, retType: type, children: body }, startPos, s.getPos()); } /** @@ -398,7 +403,7 @@ function parseFnExpr(s: ITokenStream): Ast.Node { * ``` */ function parseMatch(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.MatchKeyword); const about = parseExpr(s, false); @@ -435,7 +440,7 @@ function parseMatch(s: ITokenStream): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } @@ -463,14 +468,14 @@ function parseMatch(s: ITokenStream): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } s.nextWith(TokenKind.CloseBrace); - return NODE('match', { about, qs, default: x }, loc); + return NODE('match', { about, qs, default: x }, startPos, s.getPos()); } /** @@ -479,11 +484,12 @@ function parseMatch(s: ITokenStream): Ast.Node { * ``` */ function parseEval(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.EvalKeyword); const statements = parseBlock(s); - return NODE('block', { statements }, loc); + + return NODE('block', { statements }, startPos, s.getPos()); } /** @@ -492,11 +498,12 @@ function parseEval(s: ITokenStream): Ast.Node { * ``` */ function parseExists(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.ExistsKeyword); const identifier = parseReference(s); - return NODE('exists', { identifier }, loc); + + return NODE('exists', { identifier }, startPos, s.getPos()); } /** @@ -505,18 +512,18 @@ function parseExists(s: ITokenStream): Ast.Node { * ``` */ function parseReference(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); const segs: string[] = []; while (true) { if (segs.length > 0) { if (s.getKind() === TokenKind.Colon) { if (s.token.hasLeftSpacing) { - throw new AiScriptSyntaxError('Cannot use spaces in a reference.', s.token.loc); + throw new AiScriptSyntaxError('Cannot use spaces in a reference.', s.getPos()); } s.next(); if (s.token.hasLeftSpacing) { - throw new AiScriptSyntaxError('Cannot use spaces in a reference.', s.token.loc); + throw new AiScriptSyntaxError('Cannot use spaces in a reference.', s.getPos()); } } else { break; @@ -526,7 +533,7 @@ function parseReference(s: ITokenStream): Ast.Node { segs.push(s.token.value!); s.next(); } - return NODE('identifier', { name: segs.join(':') }, loc); + return NODE('identifier', { name: segs.join(':') }, startPos, s.getPos()); } /** @@ -535,7 +542,7 @@ function parseReference(s: ITokenStream): Ast.Node { * ``` */ function parseObject(s: ITokenStream, isStatic: boolean): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.OpenBrace); @@ -569,14 +576,14 @@ function parseObject(s: ITokenStream, isStatic: boolean): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } s.nextWith(TokenKind.CloseBrace); - return NODE('obj', { value: map }, loc); + return NODE('obj', { value: map }, startPos, s.getPos()); } /** @@ -585,7 +592,7 @@ function parseObject(s: ITokenStream, isStatic: boolean): Ast.Node { * ``` */ function parseArray(s: ITokenStream, isStatic: boolean): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.OpenBracket); @@ -611,14 +618,14 @@ function parseArray(s: ITokenStream, isStatic: boolean): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } s.nextWith(TokenKind.CloseBracket); - return NODE('arr', { value }, loc); + return NODE('arr', { value }, startPos, s.getPos()); } //#region Pratt parsing diff --git a/src/parser/syntaxes/statements.ts b/src/parser/syntaxes/statements.ts index d60b8d5b..2c50138e 100644 --- a/src/parser/syntaxes/statements.ts +++ b/src/parser/syntaxes/statements.ts @@ -14,7 +14,7 @@ import type { ITokenStream } from '../streams/token-stream.js'; * ``` */ export function parseStatement(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); switch (s.getKind()) { case TokenKind.VarKeyword: @@ -53,11 +53,11 @@ export function parseStatement(s: ITokenStream): Ast.Node { } case TokenKind.BreakKeyword: { s.next(); - return NODE('break', {}, loc); + return NODE('break', {}, startPos, s.getPos()); } case TokenKind.ContinueKeyword: { s.next(); - return NODE('continue', {}, loc); + return NODE('continue', {}, startPos, s.getPos()); } } const expr = parseExpr(s, false); @@ -78,7 +78,7 @@ export function parseDefStatement(s: ITokenStream): Ast.Node { return parseFnDef(s); } default: { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[s.getKind()]}`, s.token.loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[s.getKind()]}`, s.getPos()); } } } @@ -89,11 +89,10 @@ export function parseDefStatement(s: ITokenStream): Ast.Node { * ``` */ export function parseBlockOrStatement(s: ITokenStream): Ast.Node { - const loc = s.token.loc; - if (s.getKind() === TokenKind.OpenBrace) { + const startPos = s.getPos(); const statements = parseBlock(s); - return NODE('block', { statements }, loc); + return NODE('block', { statements }, startPos, s.getPos()); } else { return parseStatement(s); } @@ -105,7 +104,7 @@ export function parseBlockOrStatement(s: ITokenStream): Ast.Node { * ``` */ function parseVarDef(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); let mut; switch (s.getKind()) { @@ -118,7 +117,7 @@ function parseVarDef(s: ITokenStream): Ast.Node { break; } default: { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[s.getKind()]}`, s.token.loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[s.getKind()]}`, s.getPos()); } } s.next(); @@ -141,7 +140,7 @@ function parseVarDef(s: ITokenStream): Ast.Node { const expr = parseExpr(s, false); - return NODE('def', { name, varType: type, expr, mut, attr: [] }, loc); + return NODE('def', { name, varType: type, expr, mut, attr: [] }, startPos, s.getPos()); } /** @@ -150,7 +149,7 @@ function parseVarDef(s: ITokenStream): Ast.Node { * ``` */ function parseFnDef(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.At); @@ -168,16 +167,18 @@ function parseFnDef(s: ITokenStream): Ast.Node { const body = parseBlock(s); + const endPos = s.getPos(); + return NODE('def', { name, expr: NODE('fn', { args: params, retType: type, children: body, - }, loc), + }, startPos, endPos), mut: false, attr: [], - }, loc); + }, startPos, endPos); } /** @@ -186,11 +187,12 @@ function parseFnDef(s: ITokenStream): Ast.Node { * ``` */ function parseOut(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.Out); const expr = parseExpr(s, false); - return CALL_NODE('print', [expr], loc); + + return CALL_NODE('print', [expr], startPos, s.getPos()); } /** @@ -200,7 +202,7 @@ function parseOut(s: ITokenStream): Ast.Node { * ``` */ function parseEach(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); let hasParen = false; s.nextWith(TokenKind.EachKeyword); @@ -219,7 +221,7 @@ function parseEach(s: ITokenStream): Ast.Node { if (s.getKind() === TokenKind.Comma) { s.next(); } else { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } const items = parseExpr(s, false); @@ -234,11 +236,11 @@ function parseEach(s: ITokenStream): Ast.Node { var: name, items: items, for: body, - }, loc); + }, startPos, s.getPos()); } function parseFor(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); let hasParen = false; s.nextWith(TokenKind.ForKeyword); @@ -252,7 +254,7 @@ function parseFor(s: ITokenStream): Ast.Node { // range syntax s.next(); - const identLoc = s.token.loc; + const identPos = s.getPos(); s.expect(TokenKind.Identifier); const name = s.token.value!; @@ -263,13 +265,13 @@ function parseFor(s: ITokenStream): Ast.Node { s.next(); _from = parseExpr(s, false); } else { - _from = NODE('num', { value: 0 }, identLoc); + _from = NODE('num', { value: 0 }, identPos, identPos); } if (s.getKind() === TokenKind.Comma) { s.next(); } else { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } const to = parseExpr(s, false); @@ -285,7 +287,7 @@ function parseFor(s: ITokenStream): Ast.Node { from: _from, to, for: body, - }, loc); + }, startPos, s.getPos()); } else { // times syntax @@ -300,7 +302,7 @@ function parseFor(s: ITokenStream): Ast.Node { return NODE('for', { times, for: body, - }, loc); + }, startPos, s.getPos()); } } @@ -310,11 +312,12 @@ function parseFor(s: ITokenStream): Ast.Node { * ``` */ function parseReturn(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.ReturnKeyword); const expr = parseExpr(s, false); - return NODE('return', { expr }, loc); + + return NODE('return', { expr }, startPos, s.getPos()); } /** @@ -332,7 +335,7 @@ function parseStatementWithAttr(s: ITokenStream): Ast.Node { const statement = parseStatement(s); if (statement.type !== 'def') { - throw new AiScriptSyntaxError('invalid attribute.', statement.loc); + throw new AiScriptSyntaxError('invalid attribute.', statement.loc.start); } if (statement.attr != null) { statement.attr.push(...attrs); @@ -349,7 +352,7 @@ function parseStatementWithAttr(s: ITokenStream): Ast.Node { * ``` */ function parseAttr(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.OpenSharpBracket); @@ -361,12 +364,13 @@ function parseAttr(s: ITokenStream): Ast.Node { if (s.getKind() !== TokenKind.CloseBracket) { value = parseExpr(s, true); } else { - value = NODE('bool', { value: true }, loc); + const closePos = s.getPos(); + value = NODE('bool', { value: true }, closePos, closePos); } s.nextWith(TokenKind.CloseBracket); - return NODE('attr', { name, value }, loc); + return NODE('attr', { name, value }, startPos, s.getPos()); } /** @@ -375,11 +379,12 @@ function parseAttr(s: ITokenStream): Ast.Node { * ``` */ function parseLoop(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.LoopKeyword); const statements = parseBlock(s); - return NODE('loop', { statements }, loc); + + return NODE('loop', { statements }, startPos, s.getPos()); } /** @@ -388,23 +393,24 @@ function parseLoop(s: ITokenStream): Ast.Node { * ``` */ function parseDoWhile(s: ITokenStream): Ast.Node { - const doLoc = s.token.loc; + const doStartPos = s.getPos(); s.nextWith(TokenKind.DoKeyword); const body = parseBlockOrStatement(s); - const whileLoc = s.token.loc; + const whilePos = s.getPos(); s.nextWith(TokenKind.WhileKeyword); const cond = parseExpr(s, false); + const endPos = s.getPos(); return NODE('loop', { statements: [ body, NODE('if', { - cond: NODE('not', { expr: cond }, whileLoc), - then: NODE('break', {}, whileLoc), + cond: NODE('not', { expr: cond }, whilePos, endPos), + then: NODE('break', {}, endPos, endPos), elseif: [], - }, whileLoc), + }, whilePos, endPos), ], - }, doLoc); + }, doStartPos, endPos); } /** @@ -413,21 +419,22 @@ function parseDoWhile(s: ITokenStream): Ast.Node { * ``` */ function parseWhile(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.WhileKeyword); const cond = parseExpr(s, false); + const condEndPos = s.getPos(); const body = parseBlockOrStatement(s); return NODE('loop', { statements: [ NODE('if', { - cond: NODE('not', { expr: cond }, loc), - then: NODE('break', {}, loc), + cond: NODE('not', { expr: cond }, startPos, condEndPos), + then: NODE('break', {}, condEndPos, condEndPos), elseif: [], - }, loc), + }, startPos, condEndPos), body, ], - }, loc); + }, startPos, s.getPos()); } /** @@ -436,24 +443,24 @@ function parseWhile(s: ITokenStream): Ast.Node { * ``` */ function tryParseAssign(s: ITokenStream, dest: Ast.Node): Ast.Node | undefined { - const loc = s.token.loc; + const startPos = s.getPos(); // Assign switch (s.getKind()) { case TokenKind.Eq: { s.next(); const expr = parseExpr(s, false); - return NODE('assign', { dest, expr }, loc); + return NODE('assign', { dest, expr }, startPos, s.getPos()); } case TokenKind.PlusEq: { s.next(); const expr = parseExpr(s, false); - return NODE('addAssign', { dest, expr }, loc); + return NODE('addAssign', { dest, expr }, startPos, s.getPos()); } case TokenKind.MinusEq: { s.next(); const expr = parseExpr(s, false); - return NODE('subAssign', { dest, expr }, loc); + return NODE('subAssign', { dest, expr }, startPos, s.getPos()); } default: { return; diff --git a/src/parser/syntaxes/toplevel.ts b/src/parser/syntaxes/toplevel.ts index ebd32d28..a1914f0f 100644 --- a/src/parser/syntaxes/toplevel.ts +++ b/src/parser/syntaxes/toplevel.ts @@ -48,7 +48,7 @@ export function parseTopLevel(s: ITokenStream): Ast.Node[] { break; } default: { - throw new AiScriptSyntaxError('Multiple statements cannot be placed on a single line.', s.token.loc); + throw new AiScriptSyntaxError('Multiple statements cannot be placed on a single line.', s.getPos()); } } } @@ -62,7 +62,7 @@ export function parseTopLevel(s: ITokenStream): Ast.Node[] { * ``` */ export function parseNamespace(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.Colon2); @@ -104,13 +104,13 @@ export function parseNamespace(s: ITokenStream): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('Multiple statements cannot be placed on a single line.', s.token.loc); + throw new AiScriptSyntaxError('Multiple statements cannot be placed on a single line.', s.getPos()); } } } s.nextWith(TokenKind.CloseBrace); - return NODE('ns', { name, members }, loc); + return NODE('ns', { name, members }, startPos, s.getPos()); } /** @@ -119,7 +119,7 @@ export function parseNamespace(s: ITokenStream): Ast.Node { * ``` */ export function parseMeta(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.Sharp3); @@ -131,5 +131,5 @@ export function parseMeta(s: ITokenStream): Ast.Node { const value = parseExpr(s, true); - return NODE('meta', { name, value }, loc); + return NODE('meta', { name, value }, startPos, value.loc.end); } diff --git a/src/parser/token.ts b/src/parser/token.ts index 99af4099..d4bdaf49 100644 --- a/src/parser/token.ts +++ b/src/parser/token.ts @@ -109,12 +109,12 @@ export enum TokenKind { CloseBrace, } -export type TokenLocation = { column: number, line: number }; +export type TokenPosition = { column: number, line: number }; export class Token { constructor( public kind: TokenKind, - public loc: { column: number, line: number }, + public pos: TokenPosition, public hasLeftSpacing = false, /** for number literal, string literal */ public value?: string, @@ -127,6 +127,6 @@ export class Token { * - opts.value: for number literal, string literal * - opts.children: for template syntax */ -export function TOKEN(kind: TokenKind, loc: TokenLocation, opts?: { hasLeftSpacing?: boolean, value?: Token['value'], children?: Token['children'] }): Token { - return new Token(kind, loc, opts?.hasLeftSpacing, opts?.value, opts?.children); +export function TOKEN(kind: TokenKind, pos: TokenPosition, opts?: { hasLeftSpacing?: boolean, value?: Token['value'], children?: Token['children'] }): Token { + return new Token(kind, pos, opts?.hasLeftSpacing, opts?.value, opts?.children); } diff --git a/src/parser/utils.ts b/src/parser/utils.ts index 456764e5..1bd6d78f 100644 --- a/src/parser/utils.ts +++ b/src/parser/utils.ts @@ -1,19 +1,20 @@ import type * as Ast from '../node.js'; -export function NODE(type: string, params: Record, loc: { column: number, line: number }): Ast.Node { +export function NODE(type: string, params: Record, start: Ast.Pos, end: Ast.Pos): Ast.Node { const node: Record = { type }; for (const key of Object.keys(params)) { if (params[key] !== undefined) { node[key] = params[key]; } } - node.loc = loc; + node.loc = { start, end }; return node as Ast.Node; } -export function CALL_NODE(name: string, args: Ast.Node[], loc: { column: number, line: number }): Ast.Node { +export function CALL_NODE(name: string, args: Ast.Node[], start: Ast.Pos, end: Ast.Pos): Ast.Node { return NODE('call', { - target: NODE('identifier', { name }, loc), + // 糖衣構文はidentifierがソースコードに出現しないので長さ0とする。 + target: NODE('identifier', { name }, start, start), args, - }, loc); + }, start, end); } diff --git a/src/type.ts b/src/type.ts index 54cd9f0f..94e835a9 100644 --- a/src/type.ts +++ b/src/type.ts @@ -151,7 +151,7 @@ export function getTypeBySource(typeSource: Ast.TypeSource): Type { return T_GENERIC(typeSource.name, [innerType]); } } - throw new AiScriptSyntaxError(`Unknown type: '${getTypeNameBySource(typeSource)}'`, typeSource.loc); + throw new AiScriptSyntaxError(`Unknown type: '${getTypeNameBySource(typeSource)}'`, typeSource.loc.start); } else { const argTypes = typeSource.args.map(arg => getTypeBySource(arg)); return T_FN(argTypes, getTypeBySource(typeSource.result)); diff --git a/test/index.ts b/test/index.ts index b80a6c2a..5f3757cb 100644 --- a/test/index.ts +++ b/test/index.ts @@ -928,7 +928,10 @@ describe('Location', () => { assert.equal(nodes.length, 1); node = nodes[0]; if (!node.loc) assert.fail(); - assert.deepEqual(node.loc, { line: 2, column: 4 }); + assert.deepEqual(node.loc, { + start: { line: 2, column: 4 }, + end: { line: 2, column: 15 }, + }); }); test.concurrent('comment', async () => { let node: Ast.Node; @@ -942,7 +945,35 @@ describe('Location', () => { assert.equal(nodes.length, 1); node = nodes[0]; if (!node.loc) assert.fail(); - assert.deepEqual(node.loc, { line: 5, column: 3 }); + assert.deepEqual(node.loc.start, { line: 5, column: 3 }); + }); + test.concurrent('template', async () => { + let node: Ast.Node; + const parser = new Parser(); + const nodes = parser.parse(` + \`hoge{1}fuga\` + `); + assert.equal(nodes.length, 1); + node = nodes[0]; + if (!node.loc || node.type !== "tmpl") assert.fail(); + assert.deepEqual(node.loc, { + start: { line: 2, column: 4 }, + end: { line: 2, column: 17 }, + }); + assert.equal(node.tmpl.length, 3); + const [elem1, elem2, elem3] = node.tmpl as Ast.Expression[]; + assert.deepEqual(elem1.loc, { + start: { line: 2, column: 4 }, + end: { line: 2, column: 10 }, + }); + assert.deepEqual(elem2.loc, { + start: { line: 2, column: 10 }, + end: { line: 2, column: 11 }, + }); + assert.deepEqual(elem3.loc, { + start: { line: 2, column: 11 }, + end: { line: 2, column: 17 }, + }); }); }); diff --git a/test/interpreter.ts b/test/interpreter.ts index f9953705..446613c8 100644 --- a/test/interpreter.ts +++ b/test/interpreter.ts @@ -1,6 +1,7 @@ import * as assert from 'assert'; import { expect, test } from '@jest/globals'; -import { Parser, Interpreter, values, errors, utils } from '../src'; +import { Parser, Interpreter, values, errors, utils, Ast } from '../src'; + let { FN_NATIVE } = values; let { AiScriptRuntimeError, AiScriptIndexOutOfRangeError } = errors; @@ -65,26 +66,26 @@ describe('error handler', () => { }); describe('error location', () => { - const exeAndGetErrLoc = (src: string): Promise => new Promise((ok, ng) => { + const exeAndGetErrPos = (src: string): Promise => new Promise((ok, ng) => { const aiscript = new Interpreter({ emitError: FN_NATIVE((_args, _opts) => { throw Error('emitError'); }), }, { - err(e) { ok(e.loc) }, + err(e) { ok(e.pos) }, }); aiscript.exec(Parser.parse(src)).then(() => ng('error has not occured.')); }); test.concurrent('Non-aiscript Error', async () => { - return expect(exeAndGetErrLoc(`/* (の位置 + return expect(exeAndGetErrPos(`/* (の位置 */ emitError() `)).resolves.toEqual({ line: 3, column: 13}); }); test.concurrent('No "var" in namespace declaration', async () => { - return expect(exeAndGetErrLoc(`// vの位置 + return expect(exeAndGetErrPos(`// vの位置 :: Ai { let chan = 'kawaii' var kun = '!?' @@ -93,14 +94,14 @@ describe('error location', () => { }); test.concurrent('Index out of range', async () => { - return expect(exeAndGetErrLoc(`// [の位置 + return expect(exeAndGetErrPos(`// [の位置 let arr = [] arr[0] `)).resolves.toEqual({ line: 3, column: 7}); }); test.concurrent('Error in passed function', async () => { - return expect(exeAndGetErrLoc(`// /の位置 + return expect(exeAndGetErrPos(`// /の位置 [0, 1, 2].map(@(v){ 0/v }) @@ -108,7 +109,7 @@ describe('error location', () => { }); test.concurrent('No such prop', async () => { - return expect(exeAndGetErrLoc(`// .の位置 + return expect(exeAndGetErrPos(`// .の位置 [].ai `)).resolves.toEqual({ line: 2, column: 6}); }); diff --git a/test/parser.ts b/test/parser.ts index 893fff91..8fc188da 100644 --- a/test/parser.ts +++ b/test/parser.ts @@ -1,6 +1,6 @@ import * as assert from 'assert'; import { Scanner } from '../src/parser/scanner'; -import { TOKEN, TokenKind, TokenLocation } from '../src/parser/token'; +import { TOKEN, TokenKind, TokenPosition } from '../src/parser/token'; import { CharStream } from '../src/parser/streams/char-stream'; describe('CharStream', () => { @@ -77,8 +77,8 @@ describe('Scanner', () => { const stream = new Scanner(source); return stream; } - function next(stream: Scanner, kind: TokenKind, loc: TokenLocation, opts: { hasLeftSpacing?: boolean, value?: string }) { - assert.deepStrictEqual(stream.token, TOKEN(kind, loc, opts)); + function next(stream: Scanner, kind: TokenKind, pos: TokenPosition, opts: { hasLeftSpacing?: boolean, value?: string }) { + assert.deepStrictEqual(stream.token, TOKEN(kind, pos, opts)); stream.next(); }