From 9146cc26bdb85977021ab99d8527ca9c2ba75ca0 Mon Sep 17 00:00:00 2001 From: Asger F Date: Mon, 1 Oct 2018 14:36:19 +0100 Subject: [PATCH 1/4] TypeScript: test case for tokenization of template literals --- .../TypeScript/Tokens/PrettyPrintTokens.expected | 4 ++++ .../TypeScript/Tokens/PrettyPrintTokens.ql | 15 +++++++++++++++ .../TypeScript/Tokens/templateLiteralsJS.js | 4 ++++ .../TypeScript/Tokens/templateLiteralsTS.ts | 4 ++++ 4 files changed, 27 insertions(+) create mode 100644 javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected create mode 100644 javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.ql create mode 100644 javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsJS.js create mode 100644 javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsTS.ts diff --git a/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected b/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected new file mode 100644 index 000000000000..f82a9638acc8 --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected @@ -0,0 +1,4 @@ +| templateLiteralsJS.js:2 | console . log ( ` template~without~placeholders ` ) ; | +| templateLiteralsJS.js:3 | console . log ( ` template~with~placeholder~ ${ x } . ` ) ; | +| templateLiteralsTS.ts:2 | console . log ( `template~without~placeholders` ) ; | +| templateLiteralsTS.ts:3 | console . log ( `template~with~placeholder~${ x }.` ) ; | diff --git a/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.ql b/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.ql new file mode 100644 index 000000000000..e3344a5075cb --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.ql @@ -0,0 +1,15 @@ +import javascript + +Token getATokenAtLine(File file, int line) { + result.getFile() = file and + result.getLocation().getStartLine() = line +} + +bindingset[line] +string getTokenStringAtLine(File file, int line) { + result = concat(Token tok | tok = getATokenAtLine(file, line) | tok.toString().replaceAll(" ", "~") + " " order by tok.getLocation().getStartColumn()) +} + +from File file, int line +where exists(CallExpr call | call.getFile() = file and call.getLocation().getStartLine() = line) +select file.getBaseName() + ":" + line, getTokenStringAtLine(file, line) diff --git a/javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsJS.js b/javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsJS.js new file mode 100644 index 000000000000..689e596474a1 --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsJS.js @@ -0,0 +1,4 @@ +function f(x) { + console.log(`template without placeholders`); + console.log(`template with placeholder ${x}.`); +} diff --git a/javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsTS.ts b/javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsTS.ts new file mode 100644 index 000000000000..689e596474a1 --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsTS.ts @@ -0,0 +1,4 @@ +function f(x) { + console.log(`template without placeholders`); + console.log(`template with placeholder ${x}.`); +} From a199035a05e7fc5b2fe246706a2cbfdcee00ce03 Mon Sep 17 00:00:00 2001 From: Asger F Date: Mon, 1 Oct 2018 15:19:43 +0100 Subject: [PATCH 2/4] TypeScript: test case for whitespace before a rescanned token --- .../TypeScript/Tokens/PrettyPrintTokens.expected | 5 +++++ .../ql/test/library-tests/TypeScript/Tokens/regexpTS.ts | 4 ++++ .../library-tests/TypeScript/Tokens/templateLiteralsTS.ts | 3 +++ 3 files changed, 12 insertions(+) create mode 100644 javascript/ql/test/library-tests/TypeScript/Tokens/regexpTS.ts diff --git a/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected b/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected index f82a9638acc8..a12760caf5ee 100644 --- a/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected +++ b/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected @@ -1,4 +1,9 @@ +| regexpTS.ts:2 | console . log ( /foo/g ) ; | +| regexpTS.ts:3 | console . log ( /foo/g ) ; | | templateLiteralsJS.js:2 | console . log ( ` template~without~placeholders ` ) ; | | templateLiteralsJS.js:3 | console . log ( ` template~with~placeholder~ ${ x } . ` ) ; | | templateLiteralsTS.ts:2 | console . log ( `template~without~placeholders` ) ; | | templateLiteralsTS.ts:3 | console . log ( `template~with~placeholder~${ x }.` ) ; | +| templateLiteralsTS.ts:4 | console . log ( `template~with~placeholder~${ x }.` ) ; | +| templateLiteralsTS.ts:5 | console . log ( `template~with~placeholder~${ x }.` ) ; | +| templateLiteralsTS.ts:6 | console . log ( `template~with~placeholder~${ x }.` ) ; | diff --git a/javascript/ql/test/library-tests/TypeScript/Tokens/regexpTS.ts b/javascript/ql/test/library-tests/TypeScript/Tokens/regexpTS.ts new file mode 100644 index 000000000000..ca269c3db89a --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/Tokens/regexpTS.ts @@ -0,0 +1,4 @@ +function f() { + console.log(/foo/g); + console.log( /foo/g); +} diff --git a/javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsTS.ts b/javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsTS.ts index 689e596474a1..803685637b28 100644 --- a/javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsTS.ts +++ b/javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsTS.ts @@ -1,4 +1,7 @@ function f(x) { console.log(`template without placeholders`); console.log(`template with placeholder ${x}.`); + console.log(`template with placeholder ${x }.`); + console.log(`template with placeholder ${ x}.`); + console.log(`template with placeholder ${ x }.`); } From d3a1df644cea84b159a21fe7606c4ea9752f4e01 Mon Sep 17 00:00:00 2001 From: Asger F Date: Mon, 1 Oct 2018 15:22:02 +0100 Subject: [PATCH 3/4] TypeScript: test case for tokens starting with ">" --- .../TypeScript/Tokens/PrettyPrintTokens.expected | 8 ++++++++ .../library-tests/TypeScript/Tokens/greaterThanTS.ts | 11 +++++++++++ 2 files changed, 19 insertions(+) create mode 100644 javascript/ql/test/library-tests/TypeScript/Tokens/greaterThanTS.ts diff --git a/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected b/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected index a12760caf5ee..f9de49e83b9c 100644 --- a/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected +++ b/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected @@ -1,3 +1,11 @@ +| greaterThanTS.ts:2 | console . log ( x >= 1 ) ; | +| greaterThanTS.ts:3 | console . log ( x >>= 1 ) ; | +| greaterThanTS.ts:4 | console . log ( x >>>= 1 ) ; | +| greaterThanTS.ts:5 | console . log ( x >> 1 ) ; | +| greaterThanTS.ts:6 | console . log ( x >>> 1 ) ; | +| greaterThanTS.ts:8 | console . log ( x >= 1 ) ; | +| greaterThanTS.ts:9 | console . log ( x >= 1 ) ; | +| greaterThanTS.ts:10 | console . log ( x >= 1 ) ; | | regexpTS.ts:2 | console . log ( /foo/g ) ; | | regexpTS.ts:3 | console . log ( /foo/g ) ; | | templateLiteralsJS.js:2 | console . log ( ` template~without~placeholders ` ) ; | diff --git a/javascript/ql/test/library-tests/TypeScript/Tokens/greaterThanTS.ts b/javascript/ql/test/library-tests/TypeScript/Tokens/greaterThanTS.ts new file mode 100644 index 000000000000..391971677f51 --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/Tokens/greaterThanTS.ts @@ -0,0 +1,11 @@ +function f(x) { + console.log(x >= 1); + console.log(x >>= 1); + console.log(x >>>= 1); + console.log(x >> 1); + console.log(x >>> 1); + + console.log(x>=1); + console.log(x>= 1); + console.log(x >=1); +} From 057af7c8657c9484a759a59fd30d42c477f97b35 Mon Sep 17 00:00:00 2001 From: Asger F Date: Tue, 2 Oct 2018 10:42:33 +0100 Subject: [PATCH 4/4] TypeScript: add test case with mixed rescanned tokens --- .../TypeScript/Tokens/PrettyPrintTokens.expected | 4 ++++ javascript/ql/test/library-tests/TypeScript/Tokens/mixed.ts | 6 ++++++ 2 files changed, 10 insertions(+) create mode 100644 javascript/ql/test/library-tests/TypeScript/Tokens/mixed.ts diff --git a/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected b/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected index f9de49e83b9c..81e03e435923 100644 --- a/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected +++ b/javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected @@ -6,6 +6,10 @@ | greaterThanTS.ts:8 | console . log ( x >= 1 ) ; | | greaterThanTS.ts:9 | console . log ( x >= 1 ) ; | | greaterThanTS.ts:10 | console . log ( x >= 1 ) ; | +| mixed.ts:2 | console . log ( x >= 1 ) ; | +| mixed.ts:3 | console . log ( `${ /r/g >= 1 }` ) ; | +| mixed.ts:4 | console . log ( /r/g ) ; | +| mixed.ts:5 | console . log ( `${ 1 }${ 1 }` ) ; | | regexpTS.ts:2 | console . log ( /foo/g ) ; | | regexpTS.ts:3 | console . log ( /foo/g ) ; | | templateLiteralsJS.js:2 | console . log ( ` template~without~placeholders ` ) ; | diff --git a/javascript/ql/test/library-tests/TypeScript/Tokens/mixed.ts b/javascript/ql/test/library-tests/TypeScript/Tokens/mixed.ts new file mode 100644 index 000000000000..7f01ada79ad9 --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/Tokens/mixed.ts @@ -0,0 +1,6 @@ +function f(x) { + console.log(x >= 1); + console.log(`${/r/g >= 1}`); + console.log( /r/g); + console.log( `${1}${1}`); +}