From e329a42facd64bad0c383a06e8889be231d9c711 Mon Sep 17 00:00:00 2001 From: uzmoi Date: Fri, 19 Jul 2024 21:22:03 +0900 Subject: [PATCH 1/7] =?UTF-8?q?Loc=E3=81=AB=E3=83=8E=E3=83=BC=E3=83=89?= =?UTF-8?q?=E3=81=AE=E7=B5=82=E4=BA=86=E4=BD=8D=E7=BD=AE=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/error.ts | 16 ++-- src/interpreter/index.ts | 10 +-- src/node.ts | 7 +- src/parser/plugins/validate-keyword.ts | 2 +- src/parser/syntaxes/common.ts | 8 +- src/parser/syntaxes/expressions.ts | 119 +++++++++++++------------ src/parser/syntaxes/statements.ts | 91 ++++++++++--------- src/parser/syntaxes/toplevel.ts | 8 +- src/parser/utils.ts | 11 +-- src/type.ts | 2 +- test/index.ts | 7 +- test/interpreter.ts | 7 +- 12 files changed, 156 insertions(+), 132 deletions(-) diff --git a/src/error.ts b/src/error.ts index 6d66c0f5..4f1ffb53 100644 --- a/src/error.ts +++ b/src/error.ts @@ -1,11 +1,11 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import type { Loc } from './node.js'; +import type { Pos } from './node.js'; export abstract class AiScriptError extends Error { // name is read by Error.prototype.toString public name = 'AiScript'; public info?: any; - public loc?: Loc; + public pos?: Pos; constructor(message: string, info?: any) { super(message); @@ -34,8 +34,8 @@ export class NonAiScriptError extends AiScriptError { */ export class AiScriptSyntaxError extends AiScriptError { public name = 'Syntax'; - constructor(message: string, public loc: Loc, info?: any) { - super(`${message} (Line ${loc.line}, Column ${loc.column})`, info); + constructor(message: string, public pos: Pos, info?: any) { + super(`${message} (Line ${pos.line}, Column ${pos.column})`, info); } } /** @@ -43,8 +43,8 @@ export class AiScriptSyntaxError extends AiScriptError { */ export class AiScriptTypeError extends AiScriptError { public name = 'Type'; - constructor(message: string, public loc: Loc, info?: any) { - super(`${message} (Line ${loc.line}, Column ${loc.column})`, info); + constructor(message: string, public pos: Pos, info?: any) { + super(`${message} (Line ${pos.line}, Column ${pos.column})`, info); } } @@ -53,8 +53,8 @@ export class AiScriptTypeError extends AiScriptError { */ export class AiScriptNamespaceError extends AiScriptError { public name = 'Namespace'; - constructor(message: string, public loc: Loc, info?: any) { - super(`${message} (Line ${loc.line}, Column ${loc.column})`, info); + constructor(message: string, public pos: Pos, info?: any) { + super(`${message} (Line ${pos.line}, Column ${pos.column})`, info); } } diff --git a/src/interpreter/index.ts b/src/interpreter/index.ts index 38c71545..1899731e 100644 --- a/src/interpreter/index.ts +++ b/src/interpreter/index.ts @@ -191,7 +191,7 @@ export class Interpreter { switch (node.type) { case 'def': { if (node.mut) { - throw new AiScriptNamespaceError('No "var" in namespace declaration: ' + node.name, node.loc); + throw new AiScriptNamespaceError('No "var" in namespace declaration: ' + node.name, node.loc.start); } const variable: Variable = { @@ -211,7 +211,7 @@ export class Interpreter { // exhaustiveness check const n: never = node; const nd = n as Ast.Node; - throw new AiScriptNamespaceError('invalid ns member type: ' + nd.type, nd.loc); + throw new AiScriptNamespaceError('invalid ns member type: ' + nd.type, nd.loc.start); } } } @@ -246,11 +246,11 @@ export class Interpreter { @autobind private _eval(node: Ast.Node, scope: Scope): Promise { return this.__eval(node, scope).catch(e => { - if (e.loc) throw e; + if (e.pos) throw e; else { const e2 = (e instanceof AiScriptError) ? e : new NonAiScriptError(e); - e2.loc = node.loc; - e2.message = `${e2.message} (Line ${node.loc.line}, Column ${node.loc.column})`; + e2.pos = node.loc.start; + e2.message = `${e2.message} (Line ${e2.pos.line}, Column ${e2.pos.column})`; throw e2; } }); diff --git a/src/node.ts b/src/node.ts index 43b556fd..ebc4ad62 100644 --- a/src/node.ts +++ b/src/node.ts @@ -2,11 +2,16 @@ * ASTノード */ -export type Loc = { +export type Pos = { line: number; column: number; }; +export type Loc = { + start: Pos; + end: Pos; +}; + export type Node = Namespace | Meta | Statement | Expression | TypeSource | Attribute; type NodeBase = { diff --git a/src/parser/plugins/validate-keyword.ts b/src/parser/plugins/validate-keyword.ts index 97b1a3cf..d44da8f9 100644 --- a/src/parser/plugins/validate-keyword.ts +++ b/src/parser/plugins/validate-keyword.ts @@ -53,7 +53,7 @@ const reservedWord = [ ]; function throwReservedWordError(name: string, loc: Ast.Loc): void { - throw new AiScriptSyntaxError(`Reserved word "${name}" cannot be used as variable name.`, loc); + throw new AiScriptSyntaxError(`Reserved word "${name}" cannot be used as variable name.`, loc.start); } function validateNode(node: Ast.Node): Ast.Node { diff --git a/src/parser/syntaxes/common.ts b/src/parser/syntaxes/common.ts index f185a158..b23962fd 100644 --- a/src/parser/syntaxes/common.ts +++ b/src/parser/syntaxes/common.ts @@ -126,7 +126,7 @@ export function parseType(s: ITokenStream): Ast.Node { * ``` */ function parseFnType(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.At); s.nextWith(TokenKind.OpenParen); @@ -153,7 +153,7 @@ function parseFnType(s: ITokenStream): Ast.Node { const resultType = parseType(s); - return NODE('fnTypeSource', { args: params, result: resultType }, loc); + return NODE('fnTypeSource', { args: params, result: resultType }, startPos, s.token.loc); } /** @@ -162,7 +162,7 @@ function parseFnType(s: ITokenStream): Ast.Node { * ``` */ function parseNamedType(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.expect(TokenKind.Identifier); const name = s.token.value!; @@ -176,7 +176,7 @@ function parseNamedType(s: ITokenStream): Ast.Node { s.nextWith(TokenKind.Gt); } - return NODE('namedTypeSource', { name, inner }, loc); + return NODE('namedTypeSource', { name, inner }, startPos, s.token.loc); } //#endregion Type diff --git a/src/parser/syntaxes/expressions.ts b/src/parser/syntaxes/expressions.ts index 439b8fc1..023ba4ae 100644 --- a/src/parser/syntaxes/expressions.ts +++ b/src/parser/syntaxes/expressions.ts @@ -52,7 +52,7 @@ const operators: OpInfo[] = [ ]; function parsePrefix(s: ITokenStream, minBp: number): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; const op = s.getKind(); s.next(); @@ -64,38 +64,40 @@ function parsePrefix(s: ITokenStream, minBp: number): Ast.Node { const expr = parsePratt(s, minBp); + const endPos = expr.loc.end; + switch (op) { case TokenKind.Plus: { // 数値リテラル以外は非サポート if (expr.type === 'num') { - return NODE('num', { value: expr.value }, loc); + return NODE('num', { value: expr.value }, startPos, endPos); } else { - throw new AiScriptSyntaxError('currently, sign is only supported for number literal.', loc); + throw new AiScriptSyntaxError('currently, sign is only supported for number literal.', startPos); } // TODO: 将来的にサポートされる式を拡張 - // return NODE('plus', { expr }, loc); + // return NODE('plus', { expr }, startPos, endPos); } case TokenKind.Minus: { // 数値リテラル以外は非サポート if (expr.type === 'num') { - return NODE('num', { value: -1 * expr.value }, loc); + return NODE('num', { value: -1 * expr.value }, startPos, endPos); } else { - throw new AiScriptSyntaxError('currently, sign is only supported for number literal.', loc); + throw new AiScriptSyntaxError('currently, sign is only supported for number literal.', startPos); } // TODO: 将来的にサポートされる式を拡張 - // return NODE('minus', { expr }, loc); + // return NODE('minus', { expr }, startPos, endPos); } case TokenKind.Not: { - return NODE('not', { expr }, loc); + return NODE('not', { expr }, startPos, endPos); } default: { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[op]}`, loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[op]}`, startPos); } } } function parseInfix(s: ITokenStream, left: Ast.Node, minBp: number): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; const op = s.getKind(); s.next(); @@ -113,62 +115,63 @@ function parseInfix(s: ITokenStream, left: Ast.Node, minBp: number): Ast.Node { return NODE('prop', { target: left, name, - }, loc); + }, startPos, s.token.loc); } else { const right = parsePratt(s, minBp); + const endPos = s.token.loc; switch (op) { case TokenKind.Hat: { - return CALL_NODE('Core:pow', [left, right], loc); + return CALL_NODE('Core:pow', [left, right], startPos, endPos); } case TokenKind.Asterisk: { - return CALL_NODE('Core:mul', [left, right], loc); + return CALL_NODE('Core:mul', [left, right], startPos, endPos); } case TokenKind.Slash: { - return CALL_NODE('Core:div', [left, right], loc); + return CALL_NODE('Core:div', [left, right], startPos, endPos); } case TokenKind.Percent: { - return CALL_NODE('Core:mod', [left, right], loc); + return CALL_NODE('Core:mod', [left, right], startPos, endPos); } case TokenKind.Plus: { - return CALL_NODE('Core:add', [left, right], loc); + return CALL_NODE('Core:add', [left, right], startPos, endPos); } case TokenKind.Minus: { - return CALL_NODE('Core:sub', [left, right], loc); + return CALL_NODE('Core:sub', [left, right], startPos, endPos); } case TokenKind.Lt: { - return CALL_NODE('Core:lt', [left, right], loc); + return CALL_NODE('Core:lt', [left, right], startPos, endPos); } case TokenKind.LtEq: { - return CALL_NODE('Core:lteq', [left, right], loc); + return CALL_NODE('Core:lteq', [left, right], startPos, endPos); } case TokenKind.Gt: { - return CALL_NODE('Core:gt', [left, right], loc); + return CALL_NODE('Core:gt', [left, right], startPos, endPos); } case TokenKind.GtEq: { - return CALL_NODE('Core:gteq', [left, right], loc); + return CALL_NODE('Core:gteq', [left, right], startPos, endPos); } case TokenKind.Eq2: { - return CALL_NODE('Core:eq', [left, right], loc); + return CALL_NODE('Core:eq', [left, right], startPos, endPos); } case TokenKind.NotEq: { - return CALL_NODE('Core:neq', [left, right], loc); + return CALL_NODE('Core:neq', [left, right], startPos, endPos); } case TokenKind.And2: { - return NODE('and', { left, right }, loc); + return NODE('and', { left, right }, startPos, endPos); } case TokenKind.Or2: { - return NODE('or', { left, right }, loc); + return NODE('or', { left, right }, startPos, endPos); } default: { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[op]}`, loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[op]}`, startPos); } } } } function parsePostfix(s: ITokenStream, expr: Ast.Node): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; const op = s.getKind(); switch (op) { @@ -183,16 +186,16 @@ function parsePostfix(s: ITokenStream, expr: Ast.Node): Ast.Node { return NODE('index', { target: expr, index, - }, loc); + }, startPos, s.token.loc); } default: { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[op]}`, loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[op]}`, startPos); } } } function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; switch (s.getKind()) { case TokenKind.IfKeyword: { @@ -220,10 +223,12 @@ function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Node { if (isStatic) break; - for (const element of s.token.children!) { + for (const [i, element] of s.token.children!.entries()) { switch (element.kind) { case TokenKind.TemplateStringElement: { - values.push(NODE('str', { value: element.value! }, element.loc)); + // トークンの終了位置を取得するために先読み + const nextToken = s.token.children![i + 1] ?? s.lookahead(1); + values.push(NODE('str', { value: element.value! }, element.loc, nextToken.loc)); break; } case TokenKind.TemplateExprElement: { @@ -243,28 +248,28 @@ function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Node { } s.next(); - return NODE('tmpl', { tmpl: values }, loc); + return NODE('tmpl', { tmpl: values }, startPos, s.token.loc); } case TokenKind.StringLiteral: { const value = s.token.value!; s.next(); - return NODE('str', { value }, loc); + return NODE('str', { value }, startPos, s.token.loc); } case TokenKind.NumberLiteral: { // TODO: validate number value const value = Number(s.token.value!); s.next(); - return NODE('num', { value }, loc); + return NODE('num', { value }, startPos, s.token.loc); } case TokenKind.TrueKeyword: case TokenKind.FalseKeyword: { const value = (s.getKind() === TokenKind.TrueKeyword); s.next(); - return NODE('bool', { value }, loc); + return NODE('bool', { value }, startPos, s.token.loc); } case TokenKind.NullKeyword: { s.next(); - return NODE('null', { }, loc); + return NODE('null', {}, startPos, s.token.loc); } case TokenKind.OpenBrace: { return parseObject(s, isStatic); @@ -283,14 +288,14 @@ function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Node { return expr; } } - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[s.getKind()]}`, loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[s.getKind()]}`, startPos); } /** * Call = "(" [Expr *(SEP Expr) [SEP]] ")" */ function parseCall(s: ITokenStream, target: Ast.Node): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; const items: Ast.Node[] = []; s.nextWith(TokenKind.OpenParen); @@ -329,7 +334,7 @@ function parseCall(s: ITokenStream, target: Ast.Node): Ast.Node { return NODE('call', { target, args: items, - }, loc); + }, startPos, s.token.loc); } /** @@ -338,7 +343,7 @@ function parseCall(s: ITokenStream, target: Ast.Node): Ast.Node { * ``` */ function parseIf(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.IfKeyword); const cond = parseExpr(s, false); @@ -365,7 +370,7 @@ function parseIf(s: ITokenStream): Ast.Node { _else = parseBlockOrStatement(s); } - return NODE('if', { cond, then, elseif, else: _else }, loc); + return NODE('if', { cond, then, elseif, else: _else }, startPos, s.token.loc); } /** @@ -374,7 +379,7 @@ function parseIf(s: ITokenStream): Ast.Node { * ``` */ function parseFnExpr(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.At); @@ -388,7 +393,7 @@ function parseFnExpr(s: ITokenStream): Ast.Node { const body = parseBlock(s); - return NODE('fn', { args: params, retType: type, children: body }, loc); + return NODE('fn', { args: params, retType: type, children: body }, startPos, s.token.loc); } /** @@ -398,7 +403,7 @@ function parseFnExpr(s: ITokenStream): Ast.Node { * ``` */ function parseMatch(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.MatchKeyword); const about = parseExpr(s, false); @@ -470,7 +475,7 @@ function parseMatch(s: ITokenStream): Ast.Node { s.nextWith(TokenKind.CloseBrace); - return NODE('match', { about, qs, default: x }, loc); + return NODE('match', { about, qs, default: x }, startPos, s.token.loc); } /** @@ -479,11 +484,12 @@ function parseMatch(s: ITokenStream): Ast.Node { * ``` */ function parseEval(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.EvalKeyword); const statements = parseBlock(s); - return NODE('block', { statements }, loc); + + return NODE('block', { statements }, startPos, s.token.loc); } /** @@ -492,11 +498,12 @@ function parseEval(s: ITokenStream): Ast.Node { * ``` */ function parseExists(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.ExistsKeyword); const identifier = parseReference(s); - return NODE('exists', { identifier }, loc); + + return NODE('exists', { identifier }, startPos, s.token.loc); } /** @@ -505,7 +512,7 @@ function parseExists(s: ITokenStream): Ast.Node { * ``` */ function parseReference(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; const segs: string[] = []; while (true) { @@ -526,7 +533,7 @@ function parseReference(s: ITokenStream): Ast.Node { segs.push(s.token.value!); s.next(); } - return NODE('identifier', { name: segs.join(':') }, loc); + return NODE('identifier', { name: segs.join(':') }, startPos, s.token.loc); } /** @@ -535,7 +542,7 @@ function parseReference(s: ITokenStream): Ast.Node { * ``` */ function parseObject(s: ITokenStream, isStatic: boolean): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.OpenBrace); @@ -576,7 +583,7 @@ function parseObject(s: ITokenStream, isStatic: boolean): Ast.Node { s.nextWith(TokenKind.CloseBrace); - return NODE('obj', { value: map }, loc); + return NODE('obj', { value: map }, startPos, s.token.loc); } /** @@ -585,7 +592,7 @@ function parseObject(s: ITokenStream, isStatic: boolean): Ast.Node { * ``` */ function parseArray(s: ITokenStream, isStatic: boolean): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.OpenBracket); @@ -618,7 +625,7 @@ function parseArray(s: ITokenStream, isStatic: boolean): Ast.Node { s.nextWith(TokenKind.CloseBracket); - return NODE('arr', { value }, loc); + return NODE('arr', { value }, startPos, s.token.loc); } //#region Pratt parsing diff --git a/src/parser/syntaxes/statements.ts b/src/parser/syntaxes/statements.ts index d60b8d5b..02c6ff38 100644 --- a/src/parser/syntaxes/statements.ts +++ b/src/parser/syntaxes/statements.ts @@ -14,7 +14,7 @@ import type { ITokenStream } from '../streams/token-stream.js'; * ``` */ export function parseStatement(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; switch (s.getKind()) { case TokenKind.VarKeyword: @@ -53,11 +53,11 @@ export function parseStatement(s: ITokenStream): Ast.Node { } case TokenKind.BreakKeyword: { s.next(); - return NODE('break', {}, loc); + return NODE('break', {}, startPos, s.token.loc); } case TokenKind.ContinueKeyword: { s.next(); - return NODE('continue', {}, loc); + return NODE('continue', {}, startPos, s.token.loc); } } const expr = parseExpr(s, false); @@ -89,11 +89,10 @@ export function parseDefStatement(s: ITokenStream): Ast.Node { * ``` */ export function parseBlockOrStatement(s: ITokenStream): Ast.Node { - const loc = s.token.loc; - if (s.getKind() === TokenKind.OpenBrace) { + const startPos = s.token.loc; const statements = parseBlock(s); - return NODE('block', { statements }, loc); + return NODE('block', { statements }, startPos, s.token.loc); } else { return parseStatement(s); } @@ -105,7 +104,7 @@ export function parseBlockOrStatement(s: ITokenStream): Ast.Node { * ``` */ function parseVarDef(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; let mut; switch (s.getKind()) { @@ -141,7 +140,7 @@ function parseVarDef(s: ITokenStream): Ast.Node { const expr = parseExpr(s, false); - return NODE('def', { name, varType: type, expr, mut, attr: [] }, loc); + return NODE('def', { name, varType: type, expr, mut, attr: [] }, startPos, s.token.loc); } /** @@ -150,7 +149,7 @@ function parseVarDef(s: ITokenStream): Ast.Node { * ``` */ function parseFnDef(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.At); @@ -168,16 +167,18 @@ function parseFnDef(s: ITokenStream): Ast.Node { const body = parseBlock(s); + const endPos = s.token.loc; + return NODE('def', { name, expr: NODE('fn', { args: params, retType: type, children: body, - }, loc), + }, startPos, endPos), mut: false, attr: [], - }, loc); + }, startPos, endPos); } /** @@ -186,11 +187,12 @@ function parseFnDef(s: ITokenStream): Ast.Node { * ``` */ function parseOut(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.Out); const expr = parseExpr(s, false); - return CALL_NODE('print', [expr], loc); + + return CALL_NODE('print', [expr], startPos, s.token.loc); } /** @@ -200,7 +202,7 @@ function parseOut(s: ITokenStream): Ast.Node { * ``` */ function parseEach(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; let hasParen = false; s.nextWith(TokenKind.EachKeyword); @@ -234,11 +236,11 @@ function parseEach(s: ITokenStream): Ast.Node { var: name, items: items, for: body, - }, loc); + }, startPos, s.token.loc); } function parseFor(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; let hasParen = false; s.nextWith(TokenKind.ForKeyword); @@ -252,7 +254,7 @@ function parseFor(s: ITokenStream): Ast.Node { // range syntax s.next(); - const identLoc = s.token.loc; + const identPos = s.token.loc; s.expect(TokenKind.Identifier); const name = s.token.value!; @@ -263,7 +265,7 @@ function parseFor(s: ITokenStream): Ast.Node { s.next(); _from = parseExpr(s, false); } else { - _from = NODE('num', { value: 0 }, identLoc); + _from = NODE('num', { value: 0 }, identPos, identPos); } if (s.getKind() === TokenKind.Comma) { @@ -285,7 +287,7 @@ function parseFor(s: ITokenStream): Ast.Node { from: _from, to, for: body, - }, loc); + }, startPos, s.token.loc); } else { // times syntax @@ -300,7 +302,7 @@ function parseFor(s: ITokenStream): Ast.Node { return NODE('for', { times, for: body, - }, loc); + }, startPos, s.token.loc); } } @@ -310,11 +312,12 @@ function parseFor(s: ITokenStream): Ast.Node { * ``` */ function parseReturn(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.ReturnKeyword); const expr = parseExpr(s, false); - return NODE('return', { expr }, loc); + + return NODE('return', { expr }, startPos, s.token.loc); } /** @@ -332,7 +335,7 @@ function parseStatementWithAttr(s: ITokenStream): Ast.Node { const statement = parseStatement(s); if (statement.type !== 'def') { - throw new AiScriptSyntaxError('invalid attribute.', statement.loc); + throw new AiScriptSyntaxError('invalid attribute.', statement.loc.start); } if (statement.attr != null) { statement.attr.push(...attrs); @@ -349,7 +352,7 @@ function parseStatementWithAttr(s: ITokenStream): Ast.Node { * ``` */ function parseAttr(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.OpenSharpBracket); @@ -361,12 +364,13 @@ function parseAttr(s: ITokenStream): Ast.Node { if (s.getKind() !== TokenKind.CloseBracket) { value = parseExpr(s, true); } else { - value = NODE('bool', { value: true }, loc); + const closePos = s.token.loc; + value = NODE('bool', { value: true }, closePos, closePos); } s.nextWith(TokenKind.CloseBracket); - return NODE('attr', { name, value }, loc); + return NODE('attr', { name, value }, startPos, s.token.loc); } /** @@ -375,11 +379,12 @@ function parseAttr(s: ITokenStream): Ast.Node { * ``` */ function parseLoop(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.LoopKeyword); const statements = parseBlock(s); - return NODE('loop', { statements }, loc); + + return NODE('loop', { statements }, startPos, s.token.loc); } /** @@ -388,23 +393,24 @@ function parseLoop(s: ITokenStream): Ast.Node { * ``` */ function parseDoWhile(s: ITokenStream): Ast.Node { - const doLoc = s.token.loc; + const doStartPos = s.token.loc; s.nextWith(TokenKind.DoKeyword); const body = parseBlockOrStatement(s); - const whileLoc = s.token.loc; + const whilePos = s.token.loc; s.nextWith(TokenKind.WhileKeyword); const cond = parseExpr(s, false); + const endPos = s.token.loc; return NODE('loop', { statements: [ body, NODE('if', { - cond: NODE('not', { expr: cond }, whileLoc), - then: NODE('break', {}, whileLoc), + cond: NODE('not', { expr: cond }, whilePos, endPos), + then: NODE('break', {}, endPos, endPos), elseif: [], - }, whileLoc), + }, whilePos, endPos), ], - }, doLoc); + }, doStartPos, endPos); } /** @@ -413,21 +419,22 @@ function parseDoWhile(s: ITokenStream): Ast.Node { * ``` */ function parseWhile(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.WhileKeyword); const cond = parseExpr(s, false); + const condEndPos = s.token.loc; const body = parseBlockOrStatement(s); return NODE('loop', { statements: [ NODE('if', { - cond: NODE('not', { expr: cond }, loc), - then: NODE('break', {}, loc), + cond: NODE('not', { expr: cond }, startPos, condEndPos), + then: NODE('break', {}, condEndPos, condEndPos), elseif: [], - }, loc), + }, startPos, condEndPos), body, ], - }, loc); + }, startPos, s.token.loc); } /** @@ -443,17 +450,17 @@ function tryParseAssign(s: ITokenStream, dest: Ast.Node): Ast.Node | undefined { case TokenKind.Eq: { s.next(); const expr = parseExpr(s, false); - return NODE('assign', { dest, expr }, loc); + return NODE('assign', { dest, expr }, loc, s.token.loc); } case TokenKind.PlusEq: { s.next(); const expr = parseExpr(s, false); - return NODE('addAssign', { dest, expr }, loc); + return NODE('addAssign', { dest, expr }, loc, s.token.loc); } case TokenKind.MinusEq: { s.next(); const expr = parseExpr(s, false); - return NODE('subAssign', { dest, expr }, loc); + return NODE('subAssign', { dest, expr }, loc, s.token.loc); } default: { return; diff --git a/src/parser/syntaxes/toplevel.ts b/src/parser/syntaxes/toplevel.ts index ebd32d28..a6342219 100644 --- a/src/parser/syntaxes/toplevel.ts +++ b/src/parser/syntaxes/toplevel.ts @@ -62,7 +62,7 @@ export function parseTopLevel(s: ITokenStream): Ast.Node[] { * ``` */ export function parseNamespace(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.Colon2); @@ -110,7 +110,7 @@ export function parseNamespace(s: ITokenStream): Ast.Node { } s.nextWith(TokenKind.CloseBrace); - return NODE('ns', { name, members }, loc); + return NODE('ns', { name, members }, startPos, s.token.loc); } /** @@ -119,7 +119,7 @@ export function parseNamespace(s: ITokenStream): Ast.Node { * ``` */ export function parseMeta(s: ITokenStream): Ast.Node { - const loc = s.token.loc; + const startPos = s.token.loc; s.nextWith(TokenKind.Sharp3); @@ -131,5 +131,5 @@ export function parseMeta(s: ITokenStream): Ast.Node { const value = parseExpr(s, true); - return NODE('meta', { name, value }, loc); + return NODE('meta', { name, value }, startPos, value.loc.end); } diff --git a/src/parser/utils.ts b/src/parser/utils.ts index 456764e5..1bd6d78f 100644 --- a/src/parser/utils.ts +++ b/src/parser/utils.ts @@ -1,19 +1,20 @@ import type * as Ast from '../node.js'; -export function NODE(type: string, params: Record, loc: { column: number, line: number }): Ast.Node { +export function NODE(type: string, params: Record, start: Ast.Pos, end: Ast.Pos): Ast.Node { const node: Record = { type }; for (const key of Object.keys(params)) { if (params[key] !== undefined) { node[key] = params[key]; } } - node.loc = loc; + node.loc = { start, end }; return node as Ast.Node; } -export function CALL_NODE(name: string, args: Ast.Node[], loc: { column: number, line: number }): Ast.Node { +export function CALL_NODE(name: string, args: Ast.Node[], start: Ast.Pos, end: Ast.Pos): Ast.Node { return NODE('call', { - target: NODE('identifier', { name }, loc), + // 糖衣構文はidentifierがソースコードに出現しないので長さ0とする。 + target: NODE('identifier', { name }, start, start), args, - }, loc); + }, start, end); } diff --git a/src/type.ts b/src/type.ts index 54cd9f0f..94e835a9 100644 --- a/src/type.ts +++ b/src/type.ts @@ -151,7 +151,7 @@ export function getTypeBySource(typeSource: Ast.TypeSource): Type { return T_GENERIC(typeSource.name, [innerType]); } } - throw new AiScriptSyntaxError(`Unknown type: '${getTypeNameBySource(typeSource)}'`, typeSource.loc); + throw new AiScriptSyntaxError(`Unknown type: '${getTypeNameBySource(typeSource)}'`, typeSource.loc.start); } else { const argTypes = typeSource.args.map(arg => getTypeBySource(arg)); return T_FN(argTypes, getTypeBySource(typeSource.result)); diff --git a/test/index.ts b/test/index.ts index b80a6c2a..a491abb5 100644 --- a/test/index.ts +++ b/test/index.ts @@ -928,7 +928,10 @@ describe('Location', () => { assert.equal(nodes.length, 1); node = nodes[0]; if (!node.loc) assert.fail(); - assert.deepEqual(node.loc, { line: 2, column: 4 }); + assert.deepEqual(node.loc, { + start: { line: 2, column: 4 }, + end: { line: 2, column: 15 }, + }); }); test.concurrent('comment', async () => { let node: Ast.Node; @@ -942,7 +945,7 @@ describe('Location', () => { assert.equal(nodes.length, 1); node = nodes[0]; if (!node.loc) assert.fail(); - assert.deepEqual(node.loc, { line: 5, column: 3 }); + assert.deepEqual(node.loc.start, { line: 5, column: 3 }); }); }); diff --git a/test/interpreter.ts b/test/interpreter.ts index f9953705..41b64013 100644 --- a/test/interpreter.ts +++ b/test/interpreter.ts @@ -1,6 +1,7 @@ import * as assert from 'assert'; import { expect, test } from '@jest/globals'; -import { Parser, Interpreter, values, errors, utils } from '../src'; +import { Parser, Interpreter, values, errors, utils, Ast } from '../src'; + let { FN_NATIVE } = values; let { AiScriptRuntimeError, AiScriptIndexOutOfRangeError } = errors; @@ -65,13 +66,13 @@ describe('error handler', () => { }); describe('error location', () => { - const exeAndGetErrLoc = (src: string): Promise => new Promise((ok, ng) => { + const exeAndGetErrLoc = (src: string): Promise => new Promise((ok, ng) => { const aiscript = new Interpreter({ emitError: FN_NATIVE((_args, _opts) => { throw Error('emitError'); }), }, { - err(e) { ok(e.loc) }, + err(e) { ok(e.pos) }, }); aiscript.exec(Parser.parse(src)).then(() => ng('error has not occured.')); }); From 966298a35857d1e19eab15979071d31ffee430d9 Mon Sep 17 00:00:00 2001 From: uzmoi Date: Sat, 20 Jul 2024 22:07:52 +0900 Subject: [PATCH 2/7] =?UTF-8?q?ITokenStream=E3=81=ABgetPos=E3=82=92?= =?UTF-8?q?=E7=94=9F=E3=82=84=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/parser/scanner.ts | 11 ++++- src/parser/streams/token-stream.ts | 16 ++++++- src/parser/syntaxes/common.ts | 14 +++--- src/parser/syntaxes/expressions.ts | 74 ++++++++++++++--------------- src/parser/syntaxes/statements.ts | 76 +++++++++++++++--------------- src/parser/syntaxes/toplevel.ts | 10 ++-- 6 files changed, 110 insertions(+), 91 deletions(-) diff --git a/src/parser/scanner.ts b/src/parser/scanner.ts index fb2d4c17..9c28580c 100644 --- a/src/parser/scanner.ts +++ b/src/parser/scanner.ts @@ -3,7 +3,7 @@ import { CharStream } from './streams/char-stream.js'; import { TOKEN, TokenKind } from './token.js'; import type { ITokenStream } from './streams/token-stream.js'; -import type { Token } from './token.js'; +import type { Token, TokenLocation } from './token.js'; const spaceChars = [' ', '\t']; const lineBreakChars = ['\r', '\n']; @@ -42,6 +42,13 @@ export class Scanner implements ITokenStream { return this.token.kind; } + /** + * カーソル位置にあるトークンの位置情報を取得します。 + */ + public getPos(): TokenLocation { + return this.token.loc; + } + /** * カーソル位置を次のトークンへ進めます。 */ @@ -75,7 +82,7 @@ export class Scanner implements ITokenStream { */ public expect(kind: TokenKind): void { if (this.getKind() !== kind) { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[this.getKind()]}`, this.token.loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[this.getKind()]}`, this.getPos()); } } diff --git a/src/parser/streams/token-stream.ts b/src/parser/streams/token-stream.ts index 0ef2fe23..af7a41c7 100644 --- a/src/parser/streams/token-stream.ts +++ b/src/parser/streams/token-stream.ts @@ -1,6 +1,6 @@ import { AiScriptSyntaxError } from '../../error.js'; import { TOKEN, TokenKind } from '../token.js'; -import type { Token } from '../token.js'; +import type { Token, TokenLocation } from '../token.js'; /** * トークンの読み取りに関するインターフェース @@ -16,6 +16,11 @@ export interface ITokenStream { */ getKind(): TokenKind; + /** + * カーソル位置にあるトークンの位置情報を取得します。 + */ + getPos(): TokenLocation; + /** * カーソル位置を次のトークンへ進めます。 */ @@ -74,6 +79,13 @@ export class TokenStream implements ITokenStream { return this.token.kind; } + /** + * カーソル位置にあるトークンの位置情報を取得します。 + */ + public getPos(): TokenLocation { + return this.token.loc; + } + /** * カーソル位置を次のトークンへ進めます。 */ @@ -101,7 +113,7 @@ export class TokenStream implements ITokenStream { */ public expect(kind: TokenKind): void { if (this.getKind() !== kind) { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[this.getKind()]}`, this.token.loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[this.getKind()]}`, this.getPos()); } } diff --git a/src/parser/syntaxes/common.ts b/src/parser/syntaxes/common.ts index b23962fd..faf5cc5e 100644 --- a/src/parser/syntaxes/common.ts +++ b/src/parser/syntaxes/common.ts @@ -60,7 +60,7 @@ export function parseParams(s: ITokenStream): { name: string, argType?: Ast.Node break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } @@ -99,7 +99,7 @@ export function parseBlock(s: ITokenStream): Ast.Node[] { break; } default: { - throw new AiScriptSyntaxError('Multiple statements cannot be placed on a single line.', s.token.loc); + throw new AiScriptSyntaxError('Multiple statements cannot be placed on a single line.', s.getPos()); } } } @@ -126,7 +126,7 @@ export function parseType(s: ITokenStream): Ast.Node { * ``` */ function parseFnType(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.At); s.nextWith(TokenKind.OpenParen); @@ -140,7 +140,7 @@ function parseFnType(s: ITokenStream): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } @@ -153,7 +153,7 @@ function parseFnType(s: ITokenStream): Ast.Node { const resultType = parseType(s); - return NODE('fnTypeSource', { args: params, result: resultType }, startPos, s.token.loc); + return NODE('fnTypeSource', { args: params, result: resultType }, startPos, s.getPos()); } /** @@ -162,7 +162,7 @@ function parseFnType(s: ITokenStream): Ast.Node { * ``` */ function parseNamedType(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.expect(TokenKind.Identifier); const name = s.token.value!; @@ -176,7 +176,7 @@ function parseNamedType(s: ITokenStream): Ast.Node { s.nextWith(TokenKind.Gt); } - return NODE('namedTypeSource', { name, inner }, startPos, s.token.loc); + return NODE('namedTypeSource', { name, inner }, startPos, s.getPos()); } //#endregion Type diff --git a/src/parser/syntaxes/expressions.ts b/src/parser/syntaxes/expressions.ts index 023ba4ae..0e90b15c 100644 --- a/src/parser/syntaxes/expressions.ts +++ b/src/parser/syntaxes/expressions.ts @@ -52,7 +52,7 @@ const operators: OpInfo[] = [ ]; function parsePrefix(s: ITokenStream, minBp: number): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); const op = s.getKind(); s.next(); @@ -97,7 +97,7 @@ function parsePrefix(s: ITokenStream, minBp: number): Ast.Node { } function parseInfix(s: ITokenStream, left: Ast.Node, minBp: number): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); const op = s.getKind(); s.next(); @@ -115,10 +115,10 @@ function parseInfix(s: ITokenStream, left: Ast.Node, minBp: number): Ast.Node { return NODE('prop', { target: left, name, - }, startPos, s.token.loc); + }, startPos, s.getPos()); } else { const right = parsePratt(s, minBp); - const endPos = s.token.loc; + const endPos = s.getPos(); switch (op) { case TokenKind.Hat: { @@ -171,7 +171,7 @@ function parseInfix(s: ITokenStream, left: Ast.Node, minBp: number): Ast.Node { } function parsePostfix(s: ITokenStream, expr: Ast.Node): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); const op = s.getKind(); switch (op) { @@ -186,7 +186,7 @@ function parsePostfix(s: ITokenStream, expr: Ast.Node): Ast.Node { return NODE('index', { target: expr, index, - }, startPos, s.token.loc); + }, startPos, s.getPos()); } default: { throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[op]}`, startPos); @@ -195,7 +195,7 @@ function parsePostfix(s: ITokenStream, expr: Ast.Node): Ast.Node { } function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); switch (s.getKind()) { case TokenKind.IfKeyword: { @@ -248,28 +248,28 @@ function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Node { } s.next(); - return NODE('tmpl', { tmpl: values }, startPos, s.token.loc); + return NODE('tmpl', { tmpl: values }, startPos, s.getPos()); } case TokenKind.StringLiteral: { const value = s.token.value!; s.next(); - return NODE('str', { value }, startPos, s.token.loc); + return NODE('str', { value }, startPos, s.getPos()); } case TokenKind.NumberLiteral: { // TODO: validate number value const value = Number(s.token.value!); s.next(); - return NODE('num', { value }, startPos, s.token.loc); + return NODE('num', { value }, startPos, s.getPos()); } case TokenKind.TrueKeyword: case TokenKind.FalseKeyword: { const value = (s.getKind() === TokenKind.TrueKeyword); s.next(); - return NODE('bool', { value }, startPos, s.token.loc); + return NODE('bool', { value }, startPos, s.getPos()); } case TokenKind.NullKeyword: { s.next(); - return NODE('null', {}, startPos, s.token.loc); + return NODE('null', {}, startPos, s.getPos()); } case TokenKind.OpenBrace: { return parseObject(s, isStatic); @@ -295,7 +295,7 @@ function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Node { * Call = "(" [Expr *(SEP Expr) [SEP]] ")" */ function parseCall(s: ITokenStream, target: Ast.Node): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); const items: Ast.Node[] = []; s.nextWith(TokenKind.OpenParen); @@ -324,7 +324,7 @@ function parseCall(s: ITokenStream, target: Ast.Node): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } @@ -334,7 +334,7 @@ function parseCall(s: ITokenStream, target: Ast.Node): Ast.Node { return NODE('call', { target, args: items, - }, startPos, s.token.loc); + }, startPos, s.getPos()); } /** @@ -343,7 +343,7 @@ function parseCall(s: ITokenStream, target: Ast.Node): Ast.Node { * ``` */ function parseIf(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.IfKeyword); const cond = parseExpr(s, false); @@ -370,7 +370,7 @@ function parseIf(s: ITokenStream): Ast.Node { _else = parseBlockOrStatement(s); } - return NODE('if', { cond, then, elseif, else: _else }, startPos, s.token.loc); + return NODE('if', { cond, then, elseif, else: _else }, startPos, s.getPos()); } /** @@ -379,7 +379,7 @@ function parseIf(s: ITokenStream): Ast.Node { * ``` */ function parseFnExpr(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.At); @@ -393,7 +393,7 @@ function parseFnExpr(s: ITokenStream): Ast.Node { const body = parseBlock(s); - return NODE('fn', { args: params, retType: type, children: body }, startPos, s.token.loc); + return NODE('fn', { args: params, retType: type, children: body }, startPos, s.getPos()); } /** @@ -403,7 +403,7 @@ function parseFnExpr(s: ITokenStream): Ast.Node { * ``` */ function parseMatch(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.MatchKeyword); const about = parseExpr(s, false); @@ -440,7 +440,7 @@ function parseMatch(s: ITokenStream): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } @@ -468,14 +468,14 @@ function parseMatch(s: ITokenStream): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } s.nextWith(TokenKind.CloseBrace); - return NODE('match', { about, qs, default: x }, startPos, s.token.loc); + return NODE('match', { about, qs, default: x }, startPos, s.getPos()); } /** @@ -484,12 +484,12 @@ function parseMatch(s: ITokenStream): Ast.Node { * ``` */ function parseEval(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.EvalKeyword); const statements = parseBlock(s); - return NODE('block', { statements }, startPos, s.token.loc); + return NODE('block', { statements }, startPos, s.getPos()); } /** @@ -498,12 +498,12 @@ function parseEval(s: ITokenStream): Ast.Node { * ``` */ function parseExists(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.ExistsKeyword); const identifier = parseReference(s); - return NODE('exists', { identifier }, startPos, s.token.loc); + return NODE('exists', { identifier }, startPos, s.getPos()); } /** @@ -512,18 +512,18 @@ function parseExists(s: ITokenStream): Ast.Node { * ``` */ function parseReference(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); const segs: string[] = []; while (true) { if (segs.length > 0) { if (s.getKind() === TokenKind.Colon) { if (s.token.hasLeftSpacing) { - throw new AiScriptSyntaxError('Cannot use spaces in a reference.', s.token.loc); + throw new AiScriptSyntaxError('Cannot use spaces in a reference.', s.getPos()); } s.next(); if (s.token.hasLeftSpacing) { - throw new AiScriptSyntaxError('Cannot use spaces in a reference.', s.token.loc); + throw new AiScriptSyntaxError('Cannot use spaces in a reference.', s.getPos()); } } else { break; @@ -533,7 +533,7 @@ function parseReference(s: ITokenStream): Ast.Node { segs.push(s.token.value!); s.next(); } - return NODE('identifier', { name: segs.join(':') }, startPos, s.token.loc); + return NODE('identifier', { name: segs.join(':') }, startPos, s.getPos()); } /** @@ -542,7 +542,7 @@ function parseReference(s: ITokenStream): Ast.Node { * ``` */ function parseObject(s: ITokenStream, isStatic: boolean): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.OpenBrace); @@ -576,14 +576,14 @@ function parseObject(s: ITokenStream, isStatic: boolean): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } s.nextWith(TokenKind.CloseBrace); - return NODE('obj', { value: map }, startPos, s.token.loc); + return NODE('obj', { value: map }, startPos, s.getPos()); } /** @@ -592,7 +592,7 @@ function parseObject(s: ITokenStream, isStatic: boolean): Ast.Node { * ``` */ function parseArray(s: ITokenStream, isStatic: boolean): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.OpenBracket); @@ -618,14 +618,14 @@ function parseArray(s: ITokenStream, isStatic: boolean): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } } } s.nextWith(TokenKind.CloseBracket); - return NODE('arr', { value }, startPos, s.token.loc); + return NODE('arr', { value }, startPos, s.getPos()); } //#region Pratt parsing diff --git a/src/parser/syntaxes/statements.ts b/src/parser/syntaxes/statements.ts index 02c6ff38..86e2d030 100644 --- a/src/parser/syntaxes/statements.ts +++ b/src/parser/syntaxes/statements.ts @@ -14,7 +14,7 @@ import type { ITokenStream } from '../streams/token-stream.js'; * ``` */ export function parseStatement(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); switch (s.getKind()) { case TokenKind.VarKeyword: @@ -53,11 +53,11 @@ export function parseStatement(s: ITokenStream): Ast.Node { } case TokenKind.BreakKeyword: { s.next(); - return NODE('break', {}, startPos, s.token.loc); + return NODE('break', {}, startPos, s.getPos()); } case TokenKind.ContinueKeyword: { s.next(); - return NODE('continue', {}, startPos, s.token.loc); + return NODE('continue', {}, startPos, s.getPos()); } } const expr = parseExpr(s, false); @@ -78,7 +78,7 @@ export function parseDefStatement(s: ITokenStream): Ast.Node { return parseFnDef(s); } default: { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[s.getKind()]}`, s.token.loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[s.getKind()]}`, s.getPos()); } } } @@ -90,9 +90,9 @@ export function parseDefStatement(s: ITokenStream): Ast.Node { */ export function parseBlockOrStatement(s: ITokenStream): Ast.Node { if (s.getKind() === TokenKind.OpenBrace) { - const startPos = s.token.loc; + const startPos = s.getPos(); const statements = parseBlock(s); - return NODE('block', { statements }, startPos, s.token.loc); + return NODE('block', { statements }, startPos, s.getPos()); } else { return parseStatement(s); } @@ -104,7 +104,7 @@ export function parseBlockOrStatement(s: ITokenStream): Ast.Node { * ``` */ function parseVarDef(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); let mut; switch (s.getKind()) { @@ -117,7 +117,7 @@ function parseVarDef(s: ITokenStream): Ast.Node { break; } default: { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[s.getKind()]}`, s.token.loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[s.getKind()]}`, s.getPos()); } } s.next(); @@ -140,7 +140,7 @@ function parseVarDef(s: ITokenStream): Ast.Node { const expr = parseExpr(s, false); - return NODE('def', { name, varType: type, expr, mut, attr: [] }, startPos, s.token.loc); + return NODE('def', { name, varType: type, expr, mut, attr: [] }, startPos, s.getPos()); } /** @@ -149,7 +149,7 @@ function parseVarDef(s: ITokenStream): Ast.Node { * ``` */ function parseFnDef(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.At); @@ -167,7 +167,7 @@ function parseFnDef(s: ITokenStream): Ast.Node { const body = parseBlock(s); - const endPos = s.token.loc; + const endPos = s.getPos(); return NODE('def', { name, @@ -187,12 +187,12 @@ function parseFnDef(s: ITokenStream): Ast.Node { * ``` */ function parseOut(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.Out); const expr = parseExpr(s, false); - return CALL_NODE('print', [expr], startPos, s.token.loc); + return CALL_NODE('print', [expr], startPos, s.getPos()); } /** @@ -202,7 +202,7 @@ function parseOut(s: ITokenStream): Ast.Node { * ``` */ function parseEach(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); let hasParen = false; s.nextWith(TokenKind.EachKeyword); @@ -221,7 +221,7 @@ function parseEach(s: ITokenStream): Ast.Node { if (s.getKind() === TokenKind.Comma) { s.next(); } else { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } const items = parseExpr(s, false); @@ -236,11 +236,11 @@ function parseEach(s: ITokenStream): Ast.Node { var: name, items: items, for: body, - }, startPos, s.token.loc); + }, startPos, s.getPos()); } function parseFor(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); let hasParen = false; s.nextWith(TokenKind.ForKeyword); @@ -254,7 +254,7 @@ function parseFor(s: ITokenStream): Ast.Node { // range syntax s.next(); - const identPos = s.token.loc; + const identPos = s.getPos(); s.expect(TokenKind.Identifier); const name = s.token.value!; @@ -271,7 +271,7 @@ function parseFor(s: ITokenStream): Ast.Node { if (s.getKind() === TokenKind.Comma) { s.next(); } else { - throw new AiScriptSyntaxError('separator expected', s.token.loc); + throw new AiScriptSyntaxError('separator expected', s.getPos()); } const to = parseExpr(s, false); @@ -287,7 +287,7 @@ function parseFor(s: ITokenStream): Ast.Node { from: _from, to, for: body, - }, startPos, s.token.loc); + }, startPos, s.getPos()); } else { // times syntax @@ -302,7 +302,7 @@ function parseFor(s: ITokenStream): Ast.Node { return NODE('for', { times, for: body, - }, startPos, s.token.loc); + }, startPos, s.getPos()); } } @@ -312,12 +312,12 @@ function parseFor(s: ITokenStream): Ast.Node { * ``` */ function parseReturn(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.ReturnKeyword); const expr = parseExpr(s, false); - return NODE('return', { expr }, startPos, s.token.loc); + return NODE('return', { expr }, startPos, s.getPos()); } /** @@ -352,7 +352,7 @@ function parseStatementWithAttr(s: ITokenStream): Ast.Node { * ``` */ function parseAttr(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.OpenSharpBracket); @@ -364,13 +364,13 @@ function parseAttr(s: ITokenStream): Ast.Node { if (s.getKind() !== TokenKind.CloseBracket) { value = parseExpr(s, true); } else { - const closePos = s.token.loc; + const closePos = s.getPos(); value = NODE('bool', { value: true }, closePos, closePos); } s.nextWith(TokenKind.CloseBracket); - return NODE('attr', { name, value }, startPos, s.token.loc); + return NODE('attr', { name, value }, startPos, s.getPos()); } /** @@ -379,12 +379,12 @@ function parseAttr(s: ITokenStream): Ast.Node { * ``` */ function parseLoop(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.LoopKeyword); const statements = parseBlock(s); - return NODE('loop', { statements }, startPos, s.token.loc); + return NODE('loop', { statements }, startPos, s.getPos()); } /** @@ -393,13 +393,13 @@ function parseLoop(s: ITokenStream): Ast.Node { * ``` */ function parseDoWhile(s: ITokenStream): Ast.Node { - const doStartPos = s.token.loc; + const doStartPos = s.getPos(); s.nextWith(TokenKind.DoKeyword); const body = parseBlockOrStatement(s); - const whilePos = s.token.loc; + const whilePos = s.getPos(); s.nextWith(TokenKind.WhileKeyword); const cond = parseExpr(s, false); - const endPos = s.token.loc; + const endPos = s.getPos(); return NODE('loop', { statements: [ @@ -419,10 +419,10 @@ function parseDoWhile(s: ITokenStream): Ast.Node { * ``` */ function parseWhile(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.WhileKeyword); const cond = parseExpr(s, false); - const condEndPos = s.token.loc; + const condEndPos = s.getPos(); const body = parseBlockOrStatement(s); return NODE('loop', { @@ -434,7 +434,7 @@ function parseWhile(s: ITokenStream): Ast.Node { }, startPos, condEndPos), body, ], - }, startPos, s.token.loc); + }, startPos, s.getPos()); } /** @@ -443,24 +443,24 @@ function parseWhile(s: ITokenStream): Ast.Node { * ``` */ function tryParseAssign(s: ITokenStream, dest: Ast.Node): Ast.Node | undefined { - const loc = s.token.loc; + const loc = s.getPos(); // Assign switch (s.getKind()) { case TokenKind.Eq: { s.next(); const expr = parseExpr(s, false); - return NODE('assign', { dest, expr }, loc, s.token.loc); + return NODE('assign', { dest, expr }, loc, s.getPos()); } case TokenKind.PlusEq: { s.next(); const expr = parseExpr(s, false); - return NODE('addAssign', { dest, expr }, loc, s.token.loc); + return NODE('addAssign', { dest, expr }, loc, s.getPos()); } case TokenKind.MinusEq: { s.next(); const expr = parseExpr(s, false); - return NODE('subAssign', { dest, expr }, loc, s.token.loc); + return NODE('subAssign', { dest, expr }, loc, s.getPos()); } default: { return; diff --git a/src/parser/syntaxes/toplevel.ts b/src/parser/syntaxes/toplevel.ts index a6342219..a1914f0f 100644 --- a/src/parser/syntaxes/toplevel.ts +++ b/src/parser/syntaxes/toplevel.ts @@ -48,7 +48,7 @@ export function parseTopLevel(s: ITokenStream): Ast.Node[] { break; } default: { - throw new AiScriptSyntaxError('Multiple statements cannot be placed on a single line.', s.token.loc); + throw new AiScriptSyntaxError('Multiple statements cannot be placed on a single line.', s.getPos()); } } } @@ -62,7 +62,7 @@ export function parseTopLevel(s: ITokenStream): Ast.Node[] { * ``` */ export function parseNamespace(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.Colon2); @@ -104,13 +104,13 @@ export function parseNamespace(s: ITokenStream): Ast.Node { break; } default: { - throw new AiScriptSyntaxError('Multiple statements cannot be placed on a single line.', s.token.loc); + throw new AiScriptSyntaxError('Multiple statements cannot be placed on a single line.', s.getPos()); } } } s.nextWith(TokenKind.CloseBrace); - return NODE('ns', { name, members }, startPos, s.token.loc); + return NODE('ns', { name, members }, startPos, s.getPos()); } /** @@ -119,7 +119,7 @@ export function parseNamespace(s: ITokenStream): Ast.Node { * ``` */ export function parseMeta(s: ITokenStream): Ast.Node { - const startPos = s.token.loc; + const startPos = s.getPos(); s.nextWith(TokenKind.Sharp3); From 4c16df598e5533bb4d4757572c05a9923b79d539 Mon Sep 17 00:00:00 2001 From: uzmoi Date: Sat, 20 Jul 2024 22:18:50 +0900 Subject: [PATCH 3/7] rename `loc` to `pos` on `Token` --- src/parser/scanner.ts | 2 +- src/parser/streams/token-stream.ts | 2 +- src/parser/syntaxes/expressions.ts | 6 +++--- src/parser/token.ts | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/parser/scanner.ts b/src/parser/scanner.ts index 9c28580c..830176eb 100644 --- a/src/parser/scanner.ts +++ b/src/parser/scanner.ts @@ -46,7 +46,7 @@ export class Scanner implements ITokenStream { * カーソル位置にあるトークンの位置情報を取得します。 */ public getPos(): TokenLocation { - return this.token.loc; + return this.token.pos; } /** diff --git a/src/parser/streams/token-stream.ts b/src/parser/streams/token-stream.ts index af7a41c7..d4033fb7 100644 --- a/src/parser/streams/token-stream.ts +++ b/src/parser/streams/token-stream.ts @@ -83,7 +83,7 @@ export class TokenStream implements ITokenStream { * カーソル位置にあるトークンの位置情報を取得します。 */ public getPos(): TokenLocation { - return this.token.loc; + return this.token.pos; } /** diff --git a/src/parser/syntaxes/expressions.ts b/src/parser/syntaxes/expressions.ts index 0e90b15c..9fd0b893 100644 --- a/src/parser/syntaxes/expressions.ts +++ b/src/parser/syntaxes/expressions.ts @@ -228,7 +228,7 @@ function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Node { case TokenKind.TemplateStringElement: { // トークンの終了位置を取得するために先読み const nextToken = s.token.children![i + 1] ?? s.lookahead(1); - values.push(NODE('str', { value: element.value! }, element.loc, nextToken.loc)); + values.push(NODE('str', { value: element.value! }, element.pos, nextToken.pos)); break; } case TokenKind.TemplateExprElement: { @@ -236,13 +236,13 @@ function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Node { const exprStream = new TokenStream(element.children!); const expr = parseExpr(exprStream, false); if (exprStream.getKind() !== TokenKind.EOF) { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[exprStream.token.kind]}`, exprStream.token.loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[exprStream.token.kind]}`, exprStream.token.pos); } values.push(expr); break; } default: { - throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[element.kind]}`, element.loc); + throw new AiScriptSyntaxError(`unexpected token: ${TokenKind[element.kind]}`, element.pos); } } } diff --git a/src/parser/token.ts b/src/parser/token.ts index 99af4099..d1633b68 100644 --- a/src/parser/token.ts +++ b/src/parser/token.ts @@ -114,7 +114,7 @@ export type TokenLocation = { column: number, line: number }; export class Token { constructor( public kind: TokenKind, - public loc: { column: number, line: number }, + public pos: TokenLocation, public hasLeftSpacing = false, /** for number literal, string literal */ public value?: string, From b9ec1ad650618a0880911a128aff186d0244908d Mon Sep 17 00:00:00 2001 From: uzmoi Date: Sat, 20 Jul 2024 22:22:43 +0900 Subject: [PATCH 4/7] add test for tmpl location --- test/index.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/index.ts b/test/index.ts index a491abb5..5f3757cb 100644 --- a/test/index.ts +++ b/test/index.ts @@ -947,6 +947,34 @@ describe('Location', () => { if (!node.loc) assert.fail(); assert.deepEqual(node.loc.start, { line: 5, column: 3 }); }); + test.concurrent('template', async () => { + let node: Ast.Node; + const parser = new Parser(); + const nodes = parser.parse(` + \`hoge{1}fuga\` + `); + assert.equal(nodes.length, 1); + node = nodes[0]; + if (!node.loc || node.type !== "tmpl") assert.fail(); + assert.deepEqual(node.loc, { + start: { line: 2, column: 4 }, + end: { line: 2, column: 17 }, + }); + assert.equal(node.tmpl.length, 3); + const [elem1, elem2, elem3] = node.tmpl as Ast.Expression[]; + assert.deepEqual(elem1.loc, { + start: { line: 2, column: 4 }, + end: { line: 2, column: 10 }, + }); + assert.deepEqual(elem2.loc, { + start: { line: 2, column: 10 }, + end: { line: 2, column: 11 }, + }); + assert.deepEqual(elem3.loc, { + start: { line: 2, column: 11 }, + end: { line: 2, column: 17 }, + }); + }); }); describe('Unicode', () => { From eaaedbc652023397926d8be876d0a0568e9d06ee Mon Sep 17 00:00:00 2001 From: uzmoi Date: Sat, 20 Jul 2024 22:54:08 +0900 Subject: [PATCH 5/7] fix tmpl location --- src/parser/scanner.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/parser/scanner.ts b/src/parser/scanner.ts index 830176eb..b3c0d892 100644 --- a/src/parser/scanner.ts +++ b/src/parser/scanner.ts @@ -576,12 +576,14 @@ export class Scanner implements ITokenStream { } // 埋め込み式の終了 if ((this.stream.char as string) === '}') { - this.stream.next(); elements.push(TOKEN(TokenKind.TemplateExprElement, elementLoc, { hasLeftSpacing, children: tokenBuf })); - tokenBuf = []; // ここから文字列エレメントになるので位置を更新 elementLoc = this.stream.getPos(); + // TemplateExprElementトークンの終了位置をTokenStreamが取得するためのEOFトークンを追加 + tokenBuf.push(TOKEN(TokenKind.EOF, elementLoc)); + tokenBuf = []; state = 'string'; + this.stream.next(); break; } const token = this.readToken(); From a8fd9278046565f8fe5d2dfa96c75527affb0ad3 Mon Sep 17 00:00:00 2001 From: uzmoi Date: Sat, 20 Jul 2024 23:10:37 +0900 Subject: [PATCH 6/7] update api report --- etc/aiscript.api.md | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/etc/aiscript.api.md b/etc/aiscript.api.md index f0653e2d..bd332994 100644 --- a/etc/aiscript.api.md +++ b/etc/aiscript.api.md @@ -22,9 +22,9 @@ abstract class AiScriptError extends Error { // (undocumented) info?: any; // (undocumented) - loc?: Loc; - // (undocumented) name: string; + // (undocumented) + pos?: Pos; } // @public @@ -34,11 +34,11 @@ class AiScriptIndexOutOfRangeError extends AiScriptRuntimeError { // @public class AiScriptNamespaceError extends AiScriptError { - constructor(message: string, loc: Loc, info?: any); - // (undocumented) - loc: Loc; + constructor(message: string, pos: Pos, info?: any); // (undocumented) name: string; + // (undocumented) + pos: Pos; } // @public @@ -50,20 +50,20 @@ class AiScriptRuntimeError extends AiScriptError { // @public class AiScriptSyntaxError extends AiScriptError { - constructor(message: string, loc: Loc, info?: any); - // (undocumented) - loc: Loc; + constructor(message: string, pos: Pos, info?: any); // (undocumented) name: string; + // (undocumented) + pos: Pos; } // @public class AiScriptTypeError extends AiScriptError { - constructor(message: string, loc: Loc, info?: any); - // (undocumented) - loc: Loc; + constructor(message: string, pos: Pos, info?: any); // (undocumented) name: string; + // (undocumented) + pos: Pos; } // @public @@ -119,6 +119,7 @@ declare namespace Ast { export { isStatement, isExpression, + Pos, Loc, Node_2 as Node, Namespace, @@ -390,10 +391,10 @@ function isString(val: Value): val is VStr; // @public (undocumented) function jsToVal(val: any): Value; -// @public +// @public (undocumented) type Loc = { - line: number; - column: number; + start: Pos; + end: Pos; }; // @public (undocumented) @@ -503,6 +504,12 @@ export type ParserPlugin = (nodes: Ast.Node[]) => Ast.Node[]; // @public (undocumented) export type PluginType = 'validate' | 'transform'; +// @public +type Pos = { + line: number; + column: number; +}; + // @public (undocumented) type Prop = NodeBase & { type: 'prop'; From 2554b1b21d96971c97702adbfccc57b438beb59f Mon Sep 17 00:00:00 2001 From: uzmoi Date: Sat, 20 Jul 2024 23:44:05 +0900 Subject: [PATCH 7/7] =?UTF-8?q?rename=E6=BC=8F=E3=82=8C=E3=82=92=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/parser/scanner.ts | 168 ++++++++++++++--------------- src/parser/streams/token-stream.ts | 6 +- src/parser/syntaxes/statements.ts | 8 +- src/parser/token.ts | 8 +- test/interpreter.ts | 12 +-- test/parser.ts | 6 +- 6 files changed, 104 insertions(+), 104 deletions(-) diff --git a/src/parser/scanner.ts b/src/parser/scanner.ts index b3c0d892..4feb5ee7 100644 --- a/src/parser/scanner.ts +++ b/src/parser/scanner.ts @@ -3,7 +3,7 @@ import { CharStream } from './streams/char-stream.js'; import { TOKEN, TokenKind } from './token.js'; import type { ITokenStream } from './streams/token-stream.js'; -import type { Token, TokenLocation } from './token.js'; +import type { Token, TokenPosition } from './token.js'; const spaceChars = [' ', '\t']; const lineBreakChars = ['\r', '\n']; @@ -45,7 +45,7 @@ export class Scanner implements ITokenStream { /** * カーソル位置にあるトークンの位置情報を取得します。 */ - public getPos(): TokenLocation { + public getPos(): TokenPosition { return this.token.pos; } @@ -112,11 +112,11 @@ export class Scanner implements ITokenStream { } // トークン位置を記憶 - const loc = this.stream.getPos(); + const pos = this.stream.getPos(); if (lineBreakChars.includes(this.stream.char)) { this.stream.next(); - token = TOKEN(TokenKind.NewLine, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.NewLine, pos, { hasLeftSpacing }); return token; } switch (this.stream.char) { @@ -124,9 +124,9 @@ export class Scanner implements ITokenStream { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '=') { this.stream.next(); - token = TOKEN(TokenKind.NotEq, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.NotEq, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Not, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Not, pos, { hasLeftSpacing }); } break; } @@ -141,72 +141,72 @@ export class Scanner implements ITokenStream { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '#') { this.stream.next(); - token = TOKEN(TokenKind.Sharp3, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Sharp3, pos, { hasLeftSpacing }); } } else if (!this.stream.eof && (this.stream.char as string) === '[') { this.stream.next(); - token = TOKEN(TokenKind.OpenSharpBracket, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.OpenSharpBracket, pos, { hasLeftSpacing }); } else { - throw new AiScriptSyntaxError('invalid character: "#"', loc); + throw new AiScriptSyntaxError('invalid character: "#"', pos); } break; } case '%': { this.stream.next(); - token = TOKEN(TokenKind.Percent, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Percent, pos, { hasLeftSpacing }); break; } case '&': { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '&') { this.stream.next(); - token = TOKEN(TokenKind.And2, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.And2, pos, { hasLeftSpacing }); } break; } case '(': { this.stream.next(); - token = TOKEN(TokenKind.OpenParen, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.OpenParen, pos, { hasLeftSpacing }); break; } case ')': { this.stream.next(); - token = TOKEN(TokenKind.CloseParen, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.CloseParen, pos, { hasLeftSpacing }); break; } case '*': { this.stream.next(); - token = TOKEN(TokenKind.Asterisk, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Asterisk, pos, { hasLeftSpacing }); break; } case '+': { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '=') { this.stream.next(); - token = TOKEN(TokenKind.PlusEq, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.PlusEq, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Plus, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Plus, pos, { hasLeftSpacing }); } break; } case ',': { this.stream.next(); - token = TOKEN(TokenKind.Comma, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Comma, pos, { hasLeftSpacing }); break; } case '-': { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '=') { this.stream.next(); - token = TOKEN(TokenKind.MinusEq, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.MinusEq, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Minus, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Minus, pos, { hasLeftSpacing }); } break; } case '.': { this.stream.next(); - token = TOKEN(TokenKind.Dot, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Dot, pos, { hasLeftSpacing }); break; } case '/': { @@ -220,7 +220,7 @@ export class Scanner implements ITokenStream { this.skipCommentLine(); continue; } else { - token = TOKEN(TokenKind.Slash, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Slash, pos, { hasLeftSpacing }); } break; } @@ -228,27 +228,27 @@ export class Scanner implements ITokenStream { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === ':') { this.stream.next(); - token = TOKEN(TokenKind.Colon2, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Colon2, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Colon, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Colon, pos, { hasLeftSpacing }); } break; } case ';': { this.stream.next(); - token = TOKEN(TokenKind.SemiColon, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.SemiColon, pos, { hasLeftSpacing }); break; } case '<': { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '=') { this.stream.next(); - token = TOKEN(TokenKind.LtEq, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.LtEq, pos, { hasLeftSpacing }); } else if (!this.stream.eof && (this.stream.char as string) === ':') { this.stream.next(); - token = TOKEN(TokenKind.Out, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Out, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Lt, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Lt, pos, { hasLeftSpacing }); } break; } @@ -256,12 +256,12 @@ export class Scanner implements ITokenStream { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '=') { this.stream.next(); - token = TOKEN(TokenKind.Eq2, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Eq2, pos, { hasLeftSpacing }); } else if (!this.stream.eof && (this.stream.char as string) === '>') { this.stream.next(); - token = TOKEN(TokenKind.Arrow, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Arrow, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Eq, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Eq, pos, { hasLeftSpacing }); } break; } @@ -269,40 +269,40 @@ export class Scanner implements ITokenStream { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '=') { this.stream.next(); - token = TOKEN(TokenKind.GtEq, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.GtEq, pos, { hasLeftSpacing }); } else { - token = TOKEN(TokenKind.Gt, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Gt, pos, { hasLeftSpacing }); } break; } case '?': { this.stream.next(); - token = TOKEN(TokenKind.Question, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Question, pos, { hasLeftSpacing }); break; } case '@': { this.stream.next(); - token = TOKEN(TokenKind.At, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.At, pos, { hasLeftSpacing }); break; } case '[': { this.stream.next(); - token = TOKEN(TokenKind.OpenBracket, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.OpenBracket, pos, { hasLeftSpacing }); break; } case '\\': { this.stream.next(); - token = TOKEN(TokenKind.BackSlash, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.BackSlash, pos, { hasLeftSpacing }); break; } case ']': { this.stream.next(); - token = TOKEN(TokenKind.CloseBracket, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.CloseBracket, pos, { hasLeftSpacing }); break; } case '^': { this.stream.next(); - token = TOKEN(TokenKind.Hat, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Hat, pos, { hasLeftSpacing }); break; } case '`': { @@ -311,20 +311,20 @@ export class Scanner implements ITokenStream { } case '{': { this.stream.next(); - token = TOKEN(TokenKind.OpenBrace, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.OpenBrace, pos, { hasLeftSpacing }); break; } case '|': { this.stream.next(); if (!this.stream.eof && (this.stream.char as string) === '|') { this.stream.next(); - token = TOKEN(TokenKind.Or2, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.Or2, pos, { hasLeftSpacing }); } break; } case '}': { this.stream.next(); - token = TOKEN(TokenKind.CloseBrace, loc, { hasLeftSpacing }); + token = TOKEN(TokenKind.CloseBrace, pos, { hasLeftSpacing }); break; } } @@ -339,7 +339,7 @@ export class Scanner implements ITokenStream { token = wordToken; break; } - throw new AiScriptSyntaxError(`invalid character: "${this.stream.char}"`, loc); + throw new AiScriptSyntaxError(`invalid character: "${this.stream.char}"`, pos); } break; } @@ -350,7 +350,7 @@ export class Scanner implements ITokenStream { // read a word let value = ''; - const loc = this.stream.getPos(); + const pos = this.stream.getPos(); while (!this.stream.eof && wordChar.test(this.stream.char)) { value += this.stream.char; @@ -362,70 +362,70 @@ export class Scanner implements ITokenStream { // check word kind switch (value) { case 'null': { - return TOKEN(TokenKind.NullKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.NullKeyword, pos, { hasLeftSpacing }); } case 'true': { - return TOKEN(TokenKind.TrueKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.TrueKeyword, pos, { hasLeftSpacing }); } case 'false': { - return TOKEN(TokenKind.FalseKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.FalseKeyword, pos, { hasLeftSpacing }); } case 'each': { - return TOKEN(TokenKind.EachKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.EachKeyword, pos, { hasLeftSpacing }); } case 'for': { - return TOKEN(TokenKind.ForKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.ForKeyword, pos, { hasLeftSpacing }); } case 'loop': { - return TOKEN(TokenKind.LoopKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.LoopKeyword, pos, { hasLeftSpacing }); } case 'do': { - return TOKEN(TokenKind.DoKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.DoKeyword, pos, { hasLeftSpacing }); } case 'while': { - return TOKEN(TokenKind.WhileKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.WhileKeyword, pos, { hasLeftSpacing }); } case 'break': { - return TOKEN(TokenKind.BreakKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.BreakKeyword, pos, { hasLeftSpacing }); } case 'continue': { - return TOKEN(TokenKind.ContinueKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.ContinueKeyword, pos, { hasLeftSpacing }); } case 'match': { - return TOKEN(TokenKind.MatchKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.MatchKeyword, pos, { hasLeftSpacing }); } case 'case': { - return TOKEN(TokenKind.CaseKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.CaseKeyword, pos, { hasLeftSpacing }); } case 'default': { - return TOKEN(TokenKind.DefaultKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.DefaultKeyword, pos, { hasLeftSpacing }); } case 'if': { - return TOKEN(TokenKind.IfKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.IfKeyword, pos, { hasLeftSpacing }); } case 'elif': { - return TOKEN(TokenKind.ElifKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.ElifKeyword, pos, { hasLeftSpacing }); } case 'else': { - return TOKEN(TokenKind.ElseKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.ElseKeyword, pos, { hasLeftSpacing }); } case 'return': { - return TOKEN(TokenKind.ReturnKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.ReturnKeyword, pos, { hasLeftSpacing }); } case 'eval': { - return TOKEN(TokenKind.EvalKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.EvalKeyword, pos, { hasLeftSpacing }); } case 'var': { - return TOKEN(TokenKind.VarKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.VarKeyword, pos, { hasLeftSpacing }); } case 'let': { - return TOKEN(TokenKind.LetKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.LetKeyword, pos, { hasLeftSpacing }); } case 'exists': { - return TOKEN(TokenKind.ExistsKeyword, loc, { hasLeftSpacing }); + return TOKEN(TokenKind.ExistsKeyword, pos, { hasLeftSpacing }); } default: { - return TOKEN(TokenKind.Identifier, loc, { hasLeftSpacing, value }); + return TOKEN(TokenKind.Identifier, pos, { hasLeftSpacing, value }); } } } @@ -434,7 +434,7 @@ export class Scanner implements ITokenStream { let wholeNumber = ''; let fractional = ''; - const loc = this.stream.getPos(); + const pos = this.stream.getPos(); while (!this.stream.eof && digit.test(this.stream.char)) { wholeNumber += this.stream.char; @@ -450,7 +450,7 @@ export class Scanner implements ITokenStream { this.stream.next(); } if (fractional.length === 0) { - throw new AiScriptSyntaxError('digit expected', loc); + throw new AiScriptSyntaxError('digit expected', pos); } } let value; @@ -459,7 +459,7 @@ export class Scanner implements ITokenStream { } else { value = wholeNumber; } - return TOKEN(TokenKind.NumberLiteral, loc, { hasLeftSpacing, value }); + return TOKEN(TokenKind.NumberLiteral, pos, { hasLeftSpacing, value }); } private readStringLiteral(hasLeftSpacing: boolean): Token { @@ -467,14 +467,14 @@ export class Scanner implements ITokenStream { const literalMark = this.stream.char; let state: 'string' | 'escape' | 'finish' = 'string'; - const loc = this.stream.getPos(); + const pos = this.stream.getPos(); this.stream.next(); while (state !== 'finish') { switch (state) { case 'string': { if (this.stream.eof) { - throw new AiScriptSyntaxError('unexpected EOF', loc); + throw new AiScriptSyntaxError('unexpected EOF', pos); } if (this.stream.char === '\\') { this.stream.next(); @@ -492,7 +492,7 @@ export class Scanner implements ITokenStream { } case 'escape': { if (this.stream.eof) { - throw new AiScriptSyntaxError('unexpected EOF', loc); + throw new AiScriptSyntaxError('unexpected EOF', pos); } value += this.stream.char; this.stream.next(); @@ -501,7 +501,7 @@ export class Scanner implements ITokenStream { } } } - return TOKEN(TokenKind.StringLiteral, loc, { hasLeftSpacing, value }); + return TOKEN(TokenKind.StringLiteral, pos, { hasLeftSpacing, value }); } private readTemplate(hasLeftSpacing: boolean): Token { @@ -510,8 +510,8 @@ export class Scanner implements ITokenStream { let tokenBuf: Token[] = []; let state: 'string' | 'escape' | 'expr' | 'finish' = 'string'; - const loc = this.stream.getPos(); - let elementLoc = loc; + const pos = this.stream.getPos(); + let elementPos = pos; this.stream.next(); while (state !== 'finish') { @@ -519,7 +519,7 @@ export class Scanner implements ITokenStream { case 'string': { // テンプレートの終了が無いままEOFに達した if (this.stream.eof) { - throw new AiScriptSyntaxError('unexpected EOF', loc); + throw new AiScriptSyntaxError('unexpected EOF', pos); } // エスケープ if (this.stream.char === '\\') { @@ -531,7 +531,7 @@ export class Scanner implements ITokenStream { if (this.stream.char === '`') { this.stream.next(); if (buf.length > 0) { - elements.push(TOKEN(TokenKind.TemplateStringElement, elementLoc, { hasLeftSpacing, value: buf })); + elements.push(TOKEN(TokenKind.TemplateStringElement, elementPos, { hasLeftSpacing, value: buf })); } state = 'finish'; break; @@ -540,11 +540,11 @@ export class Scanner implements ITokenStream { if (this.stream.char === '{') { this.stream.next(); if (buf.length > 0) { - elements.push(TOKEN(TokenKind.TemplateStringElement, elementLoc, { hasLeftSpacing, value: buf })); + elements.push(TOKEN(TokenKind.TemplateStringElement, elementPos, { hasLeftSpacing, value: buf })); buf = ''; } // ここから式エレメントになるので位置を更新 - elementLoc = this.stream.getPos(); + elementPos = this.stream.getPos(); state = 'expr'; break; } @@ -555,7 +555,7 @@ export class Scanner implements ITokenStream { case 'escape': { // エスケープ対象の文字が無いままEOFに達した if (this.stream.eof) { - throw new AiScriptSyntaxError('unexpected EOF', loc); + throw new AiScriptSyntaxError('unexpected EOF', pos); } // 普通の文字として取り込み buf += this.stream.char; @@ -567,7 +567,7 @@ export class Scanner implements ITokenStream { case 'expr': { // 埋め込み式の終端記号が無いままEOFに達した if (this.stream.eof) { - throw new AiScriptSyntaxError('unexpected EOF', loc); + throw new AiScriptSyntaxError('unexpected EOF', pos); } // skip spasing if (spaceChars.includes(this.stream.char)) { @@ -576,11 +576,11 @@ export class Scanner implements ITokenStream { } // 埋め込み式の終了 if ((this.stream.char as string) === '}') { - elements.push(TOKEN(TokenKind.TemplateExprElement, elementLoc, { hasLeftSpacing, children: tokenBuf })); + elements.push(TOKEN(TokenKind.TemplateExprElement, elementPos, { hasLeftSpacing, children: tokenBuf })); // ここから文字列エレメントになるので位置を更新 - elementLoc = this.stream.getPos(); + elementPos = this.stream.getPos(); // TemplateExprElementトークンの終了位置をTokenStreamが取得するためのEOFトークンを追加 - tokenBuf.push(TOKEN(TokenKind.EOF, elementLoc)); + tokenBuf.push(TOKEN(TokenKind.EOF, elementPos)); tokenBuf = []; state = 'string'; this.stream.next(); @@ -593,7 +593,7 @@ export class Scanner implements ITokenStream { } } - return TOKEN(TokenKind.Template, loc, { hasLeftSpacing, children: elements }); + return TOKEN(TokenKind.Template, pos, { hasLeftSpacing, children: elements }); } private skipCommentLine(): void { diff --git a/src/parser/streams/token-stream.ts b/src/parser/streams/token-stream.ts index d4033fb7..d6d3f68b 100644 --- a/src/parser/streams/token-stream.ts +++ b/src/parser/streams/token-stream.ts @@ -1,6 +1,6 @@ import { AiScriptSyntaxError } from '../../error.js'; import { TOKEN, TokenKind } from '../token.js'; -import type { Token, TokenLocation } from '../token.js'; +import type { Token, TokenPosition } from '../token.js'; /** * トークンの読み取りに関するインターフェース @@ -19,7 +19,7 @@ export interface ITokenStream { /** * カーソル位置にあるトークンの位置情報を取得します。 */ - getPos(): TokenLocation; + getPos(): TokenPosition; /** * カーソル位置を次のトークンへ進めます。 @@ -82,7 +82,7 @@ export class TokenStream implements ITokenStream { /** * カーソル位置にあるトークンの位置情報を取得します。 */ - public getPos(): TokenLocation { + public getPos(): TokenPosition { return this.token.pos; } diff --git a/src/parser/syntaxes/statements.ts b/src/parser/syntaxes/statements.ts index 86e2d030..2c50138e 100644 --- a/src/parser/syntaxes/statements.ts +++ b/src/parser/syntaxes/statements.ts @@ -443,24 +443,24 @@ function parseWhile(s: ITokenStream): Ast.Node { * ``` */ function tryParseAssign(s: ITokenStream, dest: Ast.Node): Ast.Node | undefined { - const loc = s.getPos(); + const startPos = s.getPos(); // Assign switch (s.getKind()) { case TokenKind.Eq: { s.next(); const expr = parseExpr(s, false); - return NODE('assign', { dest, expr }, loc, s.getPos()); + return NODE('assign', { dest, expr }, startPos, s.getPos()); } case TokenKind.PlusEq: { s.next(); const expr = parseExpr(s, false); - return NODE('addAssign', { dest, expr }, loc, s.getPos()); + return NODE('addAssign', { dest, expr }, startPos, s.getPos()); } case TokenKind.MinusEq: { s.next(); const expr = parseExpr(s, false); - return NODE('subAssign', { dest, expr }, loc, s.getPos()); + return NODE('subAssign', { dest, expr }, startPos, s.getPos()); } default: { return; diff --git a/src/parser/token.ts b/src/parser/token.ts index d1633b68..d4bdaf49 100644 --- a/src/parser/token.ts +++ b/src/parser/token.ts @@ -109,12 +109,12 @@ export enum TokenKind { CloseBrace, } -export type TokenLocation = { column: number, line: number }; +export type TokenPosition = { column: number, line: number }; export class Token { constructor( public kind: TokenKind, - public pos: TokenLocation, + public pos: TokenPosition, public hasLeftSpacing = false, /** for number literal, string literal */ public value?: string, @@ -127,6 +127,6 @@ export class Token { * - opts.value: for number literal, string literal * - opts.children: for template syntax */ -export function TOKEN(kind: TokenKind, loc: TokenLocation, opts?: { hasLeftSpacing?: boolean, value?: Token['value'], children?: Token['children'] }): Token { - return new Token(kind, loc, opts?.hasLeftSpacing, opts?.value, opts?.children); +export function TOKEN(kind: TokenKind, pos: TokenPosition, opts?: { hasLeftSpacing?: boolean, value?: Token['value'], children?: Token['children'] }): Token { + return new Token(kind, pos, opts?.hasLeftSpacing, opts?.value, opts?.children); } diff --git a/test/interpreter.ts b/test/interpreter.ts index 41b64013..446613c8 100644 --- a/test/interpreter.ts +++ b/test/interpreter.ts @@ -66,7 +66,7 @@ describe('error handler', () => { }); describe('error location', () => { - const exeAndGetErrLoc = (src: string): Promise => new Promise((ok, ng) => { + const exeAndGetErrPos = (src: string): Promise => new Promise((ok, ng) => { const aiscript = new Interpreter({ emitError: FN_NATIVE((_args, _opts) => { throw Error('emitError'); @@ -78,14 +78,14 @@ describe('error location', () => { }); test.concurrent('Non-aiscript Error', async () => { - return expect(exeAndGetErrLoc(`/* (の位置 + return expect(exeAndGetErrPos(`/* (の位置 */ emitError() `)).resolves.toEqual({ line: 3, column: 13}); }); test.concurrent('No "var" in namespace declaration', async () => { - return expect(exeAndGetErrLoc(`// vの位置 + return expect(exeAndGetErrPos(`// vの位置 :: Ai { let chan = 'kawaii' var kun = '!?' @@ -94,14 +94,14 @@ describe('error location', () => { }); test.concurrent('Index out of range', async () => { - return expect(exeAndGetErrLoc(`// [の位置 + return expect(exeAndGetErrPos(`// [の位置 let arr = [] arr[0] `)).resolves.toEqual({ line: 3, column: 7}); }); test.concurrent('Error in passed function', async () => { - return expect(exeAndGetErrLoc(`// /の位置 + return expect(exeAndGetErrPos(`// /の位置 [0, 1, 2].map(@(v){ 0/v }) @@ -109,7 +109,7 @@ describe('error location', () => { }); test.concurrent('No such prop', async () => { - return expect(exeAndGetErrLoc(`// .の位置 + return expect(exeAndGetErrPos(`// .の位置 [].ai `)).resolves.toEqual({ line: 2, column: 6}); }); diff --git a/test/parser.ts b/test/parser.ts index 893fff91..8fc188da 100644 --- a/test/parser.ts +++ b/test/parser.ts @@ -1,6 +1,6 @@ import * as assert from 'assert'; import { Scanner } from '../src/parser/scanner'; -import { TOKEN, TokenKind, TokenLocation } from '../src/parser/token'; +import { TOKEN, TokenKind, TokenPosition } from '../src/parser/token'; import { CharStream } from '../src/parser/streams/char-stream'; describe('CharStream', () => { @@ -77,8 +77,8 @@ describe('Scanner', () => { const stream = new Scanner(source); return stream; } - function next(stream: Scanner, kind: TokenKind, loc: TokenLocation, opts: { hasLeftSpacing?: boolean, value?: string }) { - assert.deepStrictEqual(stream.token, TOKEN(kind, loc, opts)); + function next(stream: Scanner, kind: TokenKind, pos: TokenPosition, opts: { hasLeftSpacing?: boolean, value?: string }) { + assert.deepStrictEqual(stream.token, TOKEN(kind, pos, opts)); stream.next(); }