TypeScript: add tokenization test cases

javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.expected

@@ -0,0 +1,21 @@
+| greaterThanTS.ts:2 | console . log ( x >= 1 ) ;  |
+| greaterThanTS.ts:3 | console . log ( x >>= 1 ) ;  |
+| greaterThanTS.ts:4 | console . log ( x >>>= 1 ) ;  |
+| greaterThanTS.ts:5 | console . log ( x >> 1 ) ;  |
+| greaterThanTS.ts:6 | console . log ( x >>> 1 ) ;  |
+| greaterThanTS.ts:8 | console . log ( x >= 1 ) ;  |
+| greaterThanTS.ts:9 | console . log ( x >= 1 ) ;  |
+| greaterThanTS.ts:10 | console . log ( x >= 1 ) ;  |
+| mixed.ts:2 | console . log ( x >= 1 ) ;  |
+| mixed.ts:3 | console . log ( `${ /r/g >= 1 }` ) ;  |
+| mixed.ts:4 | console . log ( /r/g ) ;  |
+| mixed.ts:5 | console . log ( `${ 1 }${ 1 }` ) ;  |
+| regexpTS.ts:2 | console . log ( /foo/g ) ;  |
+| regexpTS.ts:3 | console . log ( /foo/g ) ;  |
+| templateLiteralsJS.js:2 | console . log ( ` template~without~placeholders ` ) ;  |
+| templateLiteralsJS.js:3 | console . log ( ` template~with~placeholder~ ${ x } . ` ) ;  |
+| templateLiteralsTS.ts:2 | console . log ( `template~without~placeholders` ) ;  |
+| templateLiteralsTS.ts:3 | console . log ( `template~with~placeholder~${ x }.` ) ;  |
+| templateLiteralsTS.ts:4 | console . log ( `template~with~placeholder~${ x }.` ) ;  |
+| templateLiteralsTS.ts:5 | console . log ( `template~with~placeholder~${ x }.` ) ;  |
+| templateLiteralsTS.ts:6 | console . log ( `template~with~placeholder~${ x }.` ) ;  |

javascript/ql/test/library-tests/TypeScript/Tokens/PrettyPrintTokens.ql

@@ -0,0 +1,15 @@
+import javascript
+
+Token getATokenAtLine(File file, int line) {
+  result.getFile() = file and
+  result.getLocation().getStartLine() = line
+}
+
+bindingset[line]
+string getTokenStringAtLine(File file, int line) {
+  result = concat(Token tok | tok = getATokenAtLine(file, line) | tok.toString().replaceAll(" ", "~") + " " order by tok.getLocation().getStartColumn())
+}
+
+from File file, int line
+where exists(CallExpr call | call.getFile() = file and call.getLocation().getStartLine() = line)
+select file.getBaseName() + ":" + line, getTokenStringAtLine(file, line)

javascript/ql/test/library-tests/TypeScript/Tokens/greaterThanTS.ts

@@ -0,0 +1,11 @@
+function f(x) {
+  console.log(x >= 1);
+  console.log(x >>= 1);
+  console.log(x >>>= 1);
+  console.log(x >> 1);
+  console.log(x >>> 1);
+
+  console.log(x>=1);
+  console.log(x>=  1);
+  console.log(x  >=1);
+}

javascript/ql/test/library-tests/TypeScript/Tokens/mixed.ts

@@ -0,0 +1,6 @@
+function f(x) {
+  console.log(x >= 1);
+  console.log(`${/r/g >= 1}`);
+  console.log( /r/g);
+  console.log( `${1}${1}`);
+}

javascript/ql/test/library-tests/TypeScript/Tokens/regexpTS.ts

@@ -0,0 +1,4 @@
+function f() {
+  console.log(/foo/g);
+  console.log(  /foo/g);
+}

javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsJS.js

@@ -0,0 +1,4 @@
+function f(x) {
+  console.log(`template without placeholders`);
+  console.log(`template with placeholder ${x}.`);
+}

javascript/ql/test/library-tests/TypeScript/Tokens/templateLiteralsTS.ts

@@ -0,0 +1,7 @@
+function f(x) {
+  console.log(`template without placeholders`);
+  console.log(`template with placeholder ${x}.`);
+  console.log(`template with placeholder ${x  }.`);
+  console.log(`template with placeholder ${  x}.`);
+  console.log(`template with placeholder ${  x  }.`);
+}