From 895b237e3cca9114c235e1e960426e040006f021 Mon Sep 17 00:00:00 2001 From: Taus Brock-Nannestad Date: Mon, 4 Feb 2019 19:03:43 +0100 Subject: [PATCH 01/71] Python: Make "Modification of parameter with default" flow-sensitive. --- change-notes/1.20/analysis-python.md | 1 + .../ModificationOfParameterWithDefault.ql | 64 ++++++++++++------- ...odificationOfParameterWithDefault.expected | 22 ++++++- .../Functions/general/functions_test.py | 10 +++ 4 files changed, 71 insertions(+), 26 deletions(-) diff --git a/change-notes/1.20/analysis-python.md b/change-notes/1.20/analysis-python.md index 442697b61762..c7ba35a66c0e 100644 --- a/change-notes/1.20/analysis-python.md +++ b/change-notes/1.20/analysis-python.md @@ -24,6 +24,7 @@ Removes false positives seen when using Python 3.6, but not when using earlier v | **Query** | **Expected impact** | **Change** | |----------------------------|------------------------|------------------------------------------------------------------| | Comparison using is when operands support \_\_eq\_\_ (`py/comparison-using-is`) | Fewer false positive results | Results where one of the objects being compared is an enum member are no longer reported. | +| Modification of parameter with default (`py/modification-of-default-value`) | More true positive results | Instances where the mutable default value is mutated inside other functions are now reported correctly. | | Unused import (`py/unused-import`) | Fewer false positive results | Results where the imported module is used in a `doctest` string are no longer reported. | | Unused import (`py/unused-import`) | Fewer false positive results | Results where the imported module is used in a type-hint comment are no longer reported. | diff --git a/python/ql/src/Functions/ModificationOfParameterWithDefault.ql b/python/ql/src/Functions/ModificationOfParameterWithDefault.ql index 03e76477dea2..2610870ee341 100644 --- a/python/ql/src/Functions/ModificationOfParameterWithDefault.ql +++ b/python/ql/src/Functions/ModificationOfParameterWithDefault.ql @@ -12,6 +12,7 @@ */ import python +import semmle.python.security.Paths predicate safe_method(string name) { name = "count" or name = "index" or name = "copy" or name = "get" or name = "has_key" or @@ -23,27 +24,6 @@ predicate maybe_parameter(SsaVariable var, Function f, Parameter p) { f.getAnArg() = p } -Name use_of_parameter(Parameter p) { - exists(SsaVariable var | - p = var.getAnUltimateDefinition().getDefinition().getNode() and - var.getAUse().getNode() = result - ) -} - -predicate modifying_call(Call c, Parameter p) { - exists(Attribute a | - c.getFunc() = a | - a.getObject() = use_of_parameter(p) and - not safe_method(a.getName()) - ) -} - -predicate is_modification(AstNode a, Parameter p) { - modifying_call(a, p) - or - a.(AugAssign).getTarget() = use_of_parameter(p) -} - predicate has_mutable_default(Parameter p) { exists(SsaVariable v, FunctionExpr f | maybe_parameter(v, f.getInnerScope(), p) and exists(int i, int def_cnt, int arg_cnt | @@ -56,6 +36,42 @@ predicate has_mutable_default(Parameter p) { ) } -from AstNode a, Parameter p -where has_mutable_default(p) and is_modification(a, p) -select a, "Modification of parameter $@, which has mutable default value.", p, p.asName().getId() +class MutableValue extends TaintKind { + MutableValue() { + this = "mutable value" + } +} + +class MutableDefaultValue extends TaintSource { + MutableDefaultValue() { + has_mutable_default(this.(NameNode).getNode()) + } + + override string toString() { + result = "mutable default value" + } + + override predicate isSourceOf(TaintKind kind) { + kind instanceof MutableValue + } +} + +class Mutation extends TaintSink { + Mutation() { + exists(AugAssign a | a.getTarget().getAFlowNode() = this) + or + exists(Call c, Attribute a | + c.getFunc() = a | + a.getObject().getAFlowNode() = this and + not safe_method(a.getName()) + ) + } + + override predicate sinks(TaintKind kind) { + kind instanceof MutableValue + } +} + +from TaintedPathSource src, TaintedPathSink sink +where src.flowsTo(sink) +select sink.getSink(), src, sink, "$@ flows to here and is mutated.", src.getSource(), "Default value" diff --git a/python/ql/test/query-tests/Functions/general/ModificationOfParameterWithDefault.expected b/python/ql/test/query-tests/Functions/general/ModificationOfParameterWithDefault.expected index 520b55ea3c27..fa8feaf186ba 100644 --- a/python/ql/test/query-tests/Functions/general/ModificationOfParameterWithDefault.expected +++ b/python/ql/test/query-tests/Functions/general/ModificationOfParameterWithDefault.expected @@ -1,2 +1,20 @@ -| functions_test.py:40:5:40:17 | Attribute() | Modification of parameter $@, which has mutable default value. | functions_test.py:39:9:39:9 | Parameter | x | -| functions_test.py:239:5:239:14 | AugAssign | Modification of parameter $@, which has mutable default value. | functions_test.py:238:15:238:15 | Parameter | x | +edges +| functions_test.py:36:9:36:9 | mutable value | functions_test.py:37:16:37:16 | mutable value | +| functions_test.py:39:9:39:9 | mutable value | functions_test.py:40:5:40:5 | mutable value | +| functions_test.py:238:15:238:15 | mutable value | functions_test.py:239:5:239:5 | mutable value | +| functions_test.py:290:25:290:25 | mutable value | functions_test.py:291:5:291:5 | mutable value | +| functions_test.py:293:21:293:21 | mutable value | functions_test.py:294:5:294:5 | mutable value | +| functions_test.py:296:27:296:27 | mutable value | functions_test.py:297:25:297:25 | mutable value | +| functions_test.py:296:27:296:27 | mutable value | functions_test.py:298:21:298:21 | mutable value | +| functions_test.py:297:25:297:25 | mutable value | functions_test.py:290:25:290:25 | mutable value | +| functions_test.py:298:21:298:21 | mutable value | functions_test.py:293:21:293:21 | mutable value | +parents +| functions_test.py:290:25:290:25 | mutable value | functions_test.py:297:25:297:25 | mutable value | +| functions_test.py:291:5:291:5 | mutable value | functions_test.py:297:25:297:25 | mutable value | +| functions_test.py:293:21:293:21 | mutable value | functions_test.py:298:21:298:21 | mutable value | +| functions_test.py:294:5:294:5 | mutable value | functions_test.py:298:21:298:21 | mutable value | +#select +| functions_test.py:40:5:40:5 | Taint sink | functions_test.py:39:9:39:9 | mutable value | functions_test.py:40:5:40:5 | mutable value | $@ flows to here and is mutated. | functions_test.py:39:9:39:9 | mutable default value | Default value | +| functions_test.py:239:5:239:5 | Taint sink | functions_test.py:238:15:238:15 | mutable value | functions_test.py:239:5:239:5 | mutable value | $@ flows to here and is mutated. | functions_test.py:238:15:238:15 | mutable default value | Default value | +| functions_test.py:291:5:291:5 | Taint sink | functions_test.py:296:27:296:27 | mutable value | functions_test.py:291:5:291:5 | mutable value | $@ flows to here and is mutated. | functions_test.py:296:27:296:27 | mutable default value | Default value | +| functions_test.py:294:5:294:5 | Taint sink | functions_test.py:296:27:296:27 | mutable value | functions_test.py:294:5:294:5 | mutable value | $@ flows to here and is mutated. | functions_test.py:296:27:296:27 | mutable default value | Default value | diff --git a/python/ql/test/query-tests/Functions/general/functions_test.py b/python/ql/test/query-tests/Functions/general/functions_test.py index 71fea62a737a..280af2bc2279 100644 --- a/python/ql/test/query-tests/Functions/general/functions_test.py +++ b/python/ql/test/query-tests/Functions/general/functions_test.py @@ -286,3 +286,13 @@ def meth(arg): Z().meth(0) +# indirect modification of parameter with default +def aug_assign_argument(x): + x += ['x'] + +def mutate_argument(x): + x.append('x') + +def indirect_modification(y = []): + aug_assign_argument(y) + mutate_argument(y) From b550da2b45f61e96816b305098b1cdac955eb05f Mon Sep 17 00:00:00 2001 From: Taus Brock-Nannestad Date: Tue, 5 Feb 2019 16:01:45 +0100 Subject: [PATCH 02/71] Improve change note. --- change-notes/1.20/analysis-python.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/change-notes/1.20/analysis-python.md b/change-notes/1.20/analysis-python.md index c7ba35a66c0e..5204b2041443 100644 --- a/change-notes/1.20/analysis-python.md +++ b/change-notes/1.20/analysis-python.md @@ -24,7 +24,7 @@ Removes false positives seen when using Python 3.6, but not when using earlier v | **Query** | **Expected impact** | **Change** | |----------------------------|------------------------|------------------------------------------------------------------| | Comparison using is when operands support \_\_eq\_\_ (`py/comparison-using-is`) | Fewer false positive results | Results where one of the objects being compared is an enum member are no longer reported. | -| Modification of parameter with default (`py/modification-of-default-value`) | More true positive results | Instances where the mutable default value is mutated inside other functions are now reported correctly. | +| Modification of parameter with default (`py/modification-of-default-value`) | More true positive results | Instances where the mutable default value is mutated inside other functions are now also reported. | | Unused import (`py/unused-import`) | Fewer false positive results | Results where the imported module is used in a `doctest` string are no longer reported. | | Unused import (`py/unused-import`) | Fewer false positive results | Results where the imported module is used in a type-hint comment are no longer reported. | From d7934276302312a10e79c33bea6b03f33d954f21 Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 13 Feb 2019 15:52:53 +0000 Subject: [PATCH 03/71] JS: treat +/- equally in suffix check query --- javascript/ql/src/Security/CWE-020/IncorrectSuffixCheck.ql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/javascript/ql/src/Security/CWE-020/IncorrectSuffixCheck.ql b/javascript/ql/src/Security/CWE-020/IncorrectSuffixCheck.ql index 6ac6519afdfc..c64fc93f5658 100644 --- a/javascript/ql/src/Security/CWE-020/IncorrectSuffixCheck.ql +++ b/javascript/ql/src/Security/CWE-020/IncorrectSuffixCheck.ql @@ -95,9 +95,9 @@ predicate isDerivedFromLength(DataFlow::Node length, DataFlow::Node operand) { or isDerivedFromLength(length.getAPredecessor(), operand) or - exists(SubExpr sub | - isDerivedFromLength(sub.getAnOperand().flow(), operand) and - length = sub.flow() + exists(BinaryExpr expr | expr instanceof SubExpr or expr instanceof AddExpr | + isDerivedFromLength(expr.getAnOperand().flow(), operand) and + length = expr.flow() ) } From 7d197692acfd4271a03ce4953bf570d9efaa32b1 Mon Sep 17 00:00:00 2001 From: Raul Garcia Date: Wed, 20 Feb 2019 17:07:04 -0800 Subject: [PATCH 04/71] Adding a new rule for detecting usage of static objects that implement ICryptoTransform that would be thread-unsafe, and potentially result in incorrect cryptographic results. --- .gitignore | 3 + .../ThreadUnSafeICryptoTransformFix.cs | 38 ++++++ .../ThreadUnsafeICryptoTransform.cs | 41 +++++++ .../ThreadUnsafeICryptoTransform.qhelp | 34 ++++++ .../ThreadUnsafeICryptoTransform.ql | 93 ++++++++++++++ .../ThreadUnsafeICryptoTransform.cs | 114 ++++++++++++++++++ .../ThreadUnsafeICryptoTransform.expected | 6 + .../ThreadUnsafeICryptoTransform.qlref | 1 + 8 files changed, 330 insertions(+) create mode 100644 csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformFix.cs create mode 100644 csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.cs create mode 100644 csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp create mode 100644 csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.ql create mode 100644 csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.cs create mode 100644 csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.expected create mode 100644 csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.qlref diff --git a/.gitignore b/.gitignore index 7e82b2f488ca..c614f8f91592 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ /.vs/ql/v15/Browse.VC.db /.vs/ProjectSettings.json +/.vs/ql_ICryptoTransform/v15/Browse.VC.opendb +/.vs/ql_ICryptoTransform/v15/Browse.VC.db +/.vs/ql_ICryptoTransform/v15/.suo diff --git a/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformFix.cs b/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformFix.cs new file mode 100644 index 000000000000..b698abdb600d --- /dev/null +++ b/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformFix.cs @@ -0,0 +1,38 @@ +internal class TokenCacheThreadUnsafeICryptoTransformDemo +{ + private static SHA256 _sha = SHA256.Create(); + + public string ComputeHash(string data) + { + byte[] passwordBytes = UTF8Encoding.UTF8.GetBytes(data); + return Convert.ToBase64String(_sha.ComputeHash(passwordBytes)); + } +} + +class Program +{ + static void Main(string[] args) + { + int max = 1000; + Task[] tasks = new Task[max]; + + Action action = (object obj) => + { + var unsafeObj = new TokenCacheThreadUnsafeICryptoTransformDemo(); + if (unsafeObj.ComputeHash((string)obj) != "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=") + { + Console.WriteLine("**** We got incorrect Results!!! ****"); + } + }; + + for (int i = 0; i < max; i++) + { + // hash calculated on all threads should be the same: + // ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= (base64) + // + tasks[i] = Task.Factory.StartNew(action, "abc"); + } + + Task.WaitAll(tasks); + } +} diff --git a/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.cs b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.cs new file mode 100644 index 000000000000..dbbc3586f981 --- /dev/null +++ b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.cs @@ -0,0 +1,41 @@ +internal class TokenCacheThreadUnsafeICryptoTransformDemoFixed +{ + // We are replacing the static SHA256 field with an instance one + // + //private static SHA256 _sha = SHA256.Create(); + private SHA256 _sha = SHA256.Create(); + + public string ComputeHash(string data) + { + byte[] passwordBytes = UTF8Encoding.UTF8.GetBytes(data); + return Convert.ToBase64String(_sha.ComputeHash(passwordBytes)); + } +} + +class Program +{ + static void Main(string[] args) + { + int max = 1000; + Task[] tasks = new Task[max]; + + Action action = (object obj) => + { + var safeObj = new TokenCacheThreadUnsafeICryptoTransformDemoFixed(); + if (safeObj.ComputeHash((string)obj) != "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=") + { + Console.WriteLine("**** We got incorrect Results!!! ****"); + } + }; + + for (int i = 0; i < max; i++) + { + // hash calculated on all threads should be the same: + // ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= (base64) + // + tasks[i] = Task.Factory.StartNew(action, "abc"); + } + + Task.WaitAll(tasks); + } +} diff --git a/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp new file mode 100644 index 000000000000..0f1ae02868d0 --- /dev/null +++ b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp @@ -0,0 +1,34 @@ + + + +

Classes that implement System.Security.Cryptography.ICryptoTransform are not thread safe.

+

Any object that implements System.Security.Cryptography.ICryptoTransform should not be used in concurrent threads as the instance members of such object are also not thread safe.

+

Potential problems may not be evident at first, but can range from explicit errors such as exceptions, to incorrect results when sharing an instance of such object in multiple threads.

+ + + +

Verify that the object is not being shared across threads.

+

If it is shared accross instances. Consider changing the code to use a non-static object of type System.Security.Cryptography.ICryptoTransform instead.

+

As an alternative, you could also look into using ThreadStatic attribute, but make sure you read the initialization remarks on the documentation.

+ + + +

This example demonstrates the dangers of using a static System.Security.Cryptography.ICryptoTransform in such a way that the results may be incorrect.

+ + +

A simple fix is to change the _sha field from being a static member to an instance one by removing the static keyword.

+ + + + +
  • + Microsoft documentation, ThreadStaticAttribute Class. +
    • +
  • + Stack Overflow, Why does SHA1.ComputeHash fail under high load with many threads?. +
    • +     + +     diff --git a/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.ql b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.ql new file mode 100644 index 000000000000..6db9fad3b45b --- /dev/null +++ b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.ql @@ -0,0 +1,93 @@ +/** + * @name Class defines a field that uses an ICryptoTransform class in a way that would be unsafe for concurrent threads. + * @description The class has a field that directly or indirectly make use of a static System.Security.Cryptography.ICryptoTransform object. + * Using this an instance of this class in concurrent threads is dangerous as it may not only result in an error, + * but under some circumstances may also result in incorrect results. + * @kind problem + * @problem.severity warning + * @precision medium + * @id cs/thread-unsafe-icryptotransform-field-in-class + * @tags concurrency + * security + * external/cwe/cwe-362 + */ + +import csharp + +class ICryptoTransform extends Class { + ICryptoTransform() { + this.getABaseType*().hasQualifiedName("System.Security.Cryptography", "ICryptoTransform") + } +} + +predicate usesICryptoTransformType( Type t ) { + exists( ICryptoTransform ict | + ict = t + or usesICryptoTransformType( t.getAChild*() ) + ) +} + +predicate hasICryptoTransformMember( Class c) { + exists( Field f | + f = c.getAMember*() + and ( + exists( ICryptoTransform ict | ict = f.getType() ) + or hasICryptoTransformMember(f.getType()) + or usesICryptoTransformType(f.getType()) + ) + ) +} + +predicate hasICryptoTransformStaticMemberNested( Class c ) { + exists( Field f | + f = c.getAMember() | + hasICryptoTransformStaticMemberNested( f.getType() ) + or ( + f.isStatic() and hasICryptoTransformMember(f.getType()) + and not exists( Attribute a + | a = f.getAnAttribute() | + a.getType().getQualifiedName() = "System.ThreadStaticAttribute" + ) + ) + ) +} + +predicate hasICryptoTransformStaticMember( Class c, string msg) { + exists( Field f | + f = c.getAMember*() + and f.isStatic() + and not exists( Attribute a + | a = f.getAnAttribute() + and a.getType().getQualifiedName() = "System.ThreadStaticAttribute" + ) + and ( + exists( ICryptoTransform ict | + ict = f.getType() + and msg = "Static field " + f + " of type " + f.getType() + ", implements 'System.Security.Cryptography.ICryptoTransform', but it does not have an attribute [ThreadStatic]. The usage of this class is unsafe for concurrent threads." + ) + or + ( + not exists( ICryptoTransform ict | ict = f.getType() ) // Avoid dup messages + and exists( Type t | t = f.getType() | + usesICryptoTransformType(t) + and msg = "Static field " + f + " of type " + f.getType() + " makes usage of 'System.Security.Cryptography.ICryptoTransform', but it does not have an attribute [ThreadStatic]. The usage of this class is unsafe for concurrent threads." + ) + ) + ) + ) + or + exists( Field f | + f = c.getAMember*() + and not f.isStatic() + and ( hasICryptoTransformStaticMember( f.getType(), _ ) + and msg = "Non-static field " + f + " of type " + f.getType() + " internally makes use of an static object that implements 'System.Security.Cryptography.ICryptoTransform'. This causes that usage of this class member is unsafe for concurrent threads." + ) + ) + or ( hasICryptoTransformStaticMemberNested(c) + and msg = "Class" + c + " implementation depends on a static object of type 'System.Security.Cryptography.ICryptoTransform' in a way that is unsafe for concurrent threads." + ) +} + +from Class c , string s + where hasICryptoTransformStaticMember(c, s) +select c, s diff --git a/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.cs b/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.cs new file mode 100644 index 000000000000..90214fc988a2 --- /dev/null +++ b/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.cs @@ -0,0 +1,114 @@ +// semmle-extractor-options: /r:System.Security.Cryptography.Csp.dll /r:System.Security.Cryptography.Algorithms.dll /r:System.Security.Cryptography.Primitives.dll +using System; +using System.Collections.Generic; +using System.Security.Cryptography; + +public class Nest01 +{ + private readonly SHA256 _sha; + + public Nest01() + { + _sha = SHA256.Create(); + } +} + +public class Nest02 +{ + private readonly Nest01 _n; + + public Nest02() + { + _n = new Nest01(); + } +} + +public class ListNonStatic +{ + private List _shaList; + + public ListNonStatic() + { + _shaList = new List(); + } +} + +/// +/// Positive results (classes are not thread safe) +/// +public class Nest03 +{ + private static readonly Nest01 _n = new Nest01(); +} + +public class Nest04 +{ + static ListNonStatic _list = new ListNonStatic(); +} + +public static class StaticMemberChildUsage +{ + public enum DigestAlgorithm + { + SHA1, + SHA256, + } + + private static readonly Dictionary HashMap = new Dictionary + { + { DigestAlgorithm.SHA1, SHA1.Create() }, + { DigestAlgorithm.SHA256, SHA256.Create() }, + }; +} + +public class StaticMember +{ + private static SHA1 _sha1 = SHA1.Create(); +} + +public class IndirectStatic +{ + StaticMember tc; +} + +public class IndirectStatic2 +{ + static Nest02 _n = new Nest02(); +} + +/// +/// Should not be flagged (thread safe) +/// + +public class TokenCacheFP +{ + /// + /// Should be OK. Not shared between threads + /// + [ThreadStatic] + private static SHA1 _sha1 = SHA1.Create(); + + private string ComputeHash(string password) + { + return password; + } +} + +public class TokenCacheNonStat +{ + /// + /// Should be OK. Not shared between threads + /// + private SHA1 _sha1; + + public TokenCacheNonStat() + { + _sha1 = SHA1.Create(); + } + + private string ComputeHash(string password) + { + return password; + } +} + diff --git a/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.expected b/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.expected new file mode 100644 index 000000000000..e9ca1952f635 --- /dev/null +++ b/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.expected @@ -0,0 +1,6 @@ +| ThreadUnsafeICryptoTransform.cs:39:14:39:19 | Nest03 | ClassNest03 implementation depends on a static object of type 'System.Security.Cryptography.ICryptoTransform' in a way that is unsafe for concurrent threads. | +| ThreadUnsafeICryptoTransform.cs:44:14:44:19 | Nest04 | ClassNest04 implementation depends on a static object of type 'System.Security.Cryptography.ICryptoTransform' in a way that is unsafe for concurrent threads. | +| ThreadUnsafeICryptoTransform.cs:49:21:49:42 | StaticMemberChildUsage | Static field HashMap of type Dictionary makes usage of 'System.Security.Cryptography.ICryptoTransform', but it does not have an attribute [ThreadStatic]. The usage of this class is unsafe for concurrent threads. | +| ThreadUnsafeICryptoTransform.cs:64:14:64:25 | StaticMember | Static field _sha1 of type SHA1, implements 'System.Security.Cryptography.ICryptoTransform', but it does not have an attribute [ThreadStatic]. The usage of this class is unsafe for concurrent threads. | +| ThreadUnsafeICryptoTransform.cs:69:14:69:27 | IndirectStatic | Non-static field tc of type StaticMember internally makes use of an static object that implements 'System.Security.Cryptography.ICryptoTransform'. This causes that usage of this class member is unsafe for concurrent threads. | +| ThreadUnsafeICryptoTransform.cs:74:14:74:28 | IndirectStatic2 | ClassIndirectStatic2 implementation depends on a static object of type 'System.Security.Cryptography.ICryptoTransform' in a way that is unsafe for concurrent threads. | diff --git a/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.qlref b/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.qlref new file mode 100644 index 000000000000..e247961a538d --- /dev/null +++ b/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.qlref @@ -0,0 +1 @@ +Likely Bugs/ThreadUnsafeICryptoTransform.ql \ No newline at end of file From fa73b8488aee98f925b8267baf5531b416d63ede Mon Sep 17 00:00:00 2001 From: Raul Garcia <42392023+raulgarciamsft@users.noreply.github.com> Date: Wed, 20 Feb 2019 17:10:19 -0800 Subject: [PATCH 05/71] Update .gitignore --- .gitignore | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitignore b/.gitignore index c614f8f91592..4b055e55a091 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,3 @@ /.vs/ql/v15/Browse.VC.opendb /.vs/ql/v15/Browse.VC.db /.vs/ProjectSettings.json - -/.vs/ql_ICryptoTransform/v15/Browse.VC.opendb -/.vs/ql_ICryptoTransform/v15/Browse.VC.db -/.vs/ql_ICryptoTransform/v15/.suo From 143b1e576ef40b3918e9227fa23058cdd2bc9d14 Mon Sep 17 00:00:00 2001 From: Raul Garcia <42392023+raulgarciamsft@users.noreply.github.com> Date: Wed, 20 Feb 2019 17:10:32 -0800 Subject: [PATCH 06/71] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4b055e55a091..7e82b2f488ca 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ /.vs/ql/v15/Browse.VC.opendb /.vs/ql/v15/Browse.VC.db /.vs/ProjectSettings.json + From 9ac8d60636f6183ed56daab7b41a28d255a7a9ab Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Mon, 4 Feb 2019 10:19:20 +0100 Subject: [PATCH 07/71] C++: IR query for redundant null check This new query is not written because it's the most interesting query we could write but because it's an IR-based query whose results are easy to verify. --- .../Likely Bugs/RedundantNullCheckSimple.ql | 72 +++++++++++++++++++ .../RedundantNullCheckSimple.cpp | 33 +++++++++ .../RedundantNullCheckSimple.expected | 2 + .../RedundantNullCheckSimple.qlref | 1 + 4 files changed, 108 insertions(+) create mode 100644 cpp/ql/src/Likely Bugs/RedundantNullCheckSimple.ql create mode 100644 cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.cpp create mode 100644 cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.expected create mode 100644 cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.qlref diff --git a/cpp/ql/src/Likely Bugs/RedundantNullCheckSimple.ql b/cpp/ql/src/Likely Bugs/RedundantNullCheckSimple.ql new file mode 100644 index 000000000000..1f96bc691d05 --- /dev/null +++ b/cpp/ql/src/Likely Bugs/RedundantNullCheckSimple.ql @@ -0,0 +1,72 @@ +/** + * @name Redundant null check due to previous dereference + * @description Checking a pointer for nullness after dereferencing it is + * likely to be a sign that either the check can be removed, or + * it should be moved before the dereference. + * @kind problem + * @problem.severity error + * @id cpp/redundant-null-check-simple + * @tags reliability + * correctness + * external/cwe/cwe-476 + */ + +/* + * Note: this query is not assigned a precision yet because we don't want it on + * LGTM until its performance is well understood. It's also lacking qhelp. + */ + +import semmle.code.cpp.ir.IR + +class NullInstruction extends ConstantValueInstruction { + NullInstruction() { + this.getValue() = "0" and + this.getResultType().getUnspecifiedType() instanceof PointerType + } +} + +/** + * An instruction that will never have slicing on its result. + */ +class SingleValuedInstruction extends Instruction { + SingleValuedInstruction() { + this.getResultMemoryAccess() instanceof IndirectMemoryAccess + or + not this.hasMemoryResult() + } +} + +predicate explicitNullTestOfInstruction(Instruction checked, Instruction bool) { + bool = any(CompareInstruction cmp | + exists(NullInstruction null | + cmp.getLeft() = null and cmp.getRight() = checked + or + cmp.getLeft() = checked and cmp.getRight() = null + | + cmp instanceof CompareEQInstruction + or + cmp instanceof CompareNEInstruction + ) + ) + or + bool = any(ConvertInstruction convert | + checked = convert.getUnary() and + convert.getResultType() instanceof BoolType and + checked.getResultType() instanceof PointerType + ) +} + +from LoadInstruction checked, LoadInstruction deref, SingleValuedInstruction sourceValue +where + explicitNullTestOfInstruction(checked, _) and + sourceValue = deref.getSourceAddress().(LoadInstruction).getSourceValue() and + sourceValue = checked.getSourceValue() and + // This also holds if the blocks are equal, meaning that the check could come + // before the deref. That's still not okay because when they're in the same + // basic block then the deref is unavoidable even if the check concluded that + // the pointer was null. To follow this idea to its full generality, we + // should also give an alert when `check` post-dominates `deref`. + deref.getBlock().dominates(checked.getBlock()) and + not checked.getAST().isInMacroExpansion() +select checked, "This null check is redundant because the value is $@ in any case", deref, + "dereferenced here" diff --git a/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.cpp b/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.cpp new file mode 100644 index 000000000000..d693e50ac598 --- /dev/null +++ b/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.cpp @@ -0,0 +1,33 @@ +void test1(int *p) { + int x; + x = *p; + if (p == nullptr) { // BAD + return; + } +} + +void test2(int *p) { + int x = *p; + if (x > 100) + return; + if (!p) // BAD + return; +} + +void test_indirect(int **p) { + int x; + x = **p; + if (*p == nullptr) { // BAD [NOT DETECTED] + return; + } +} + +struct ContainsIntPtr { + int **intPtr; +}; + +bool check_curslist(ContainsIntPtr *cip) { + // both the deref and the null check come from the same instruction, but it's + // an AliasedDefinition instruction. + return *cip->intPtr != nullptr; // GOOD +} diff --git a/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.expected b/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.expected new file mode 100644 index 000000000000..c0ece0bfa815 --- /dev/null +++ b/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.expected @@ -0,0 +1,2 @@ +| RedundantNullCheckSimple.cpp:4:7:4:7 | Load: p | This null check is redundant because the value is $@ in any case | RedundantNullCheckSimple.cpp:3:7:3:8 | Load: * ... | dereferenced here | +| RedundantNullCheckSimple.cpp:13:8:13:8 | Load: p | This null check is redundant because the value is $@ in any case | RedundantNullCheckSimple.cpp:10:11:10:12 | Load: * ... | dereferenced here | diff --git a/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.qlref b/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.qlref new file mode 100644 index 000000000000..2223e47c30d2 --- /dev/null +++ b/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.qlref @@ -0,0 +1 @@ +Likely Bugs/RedundantNullCheckSimple.ql From 12084fc90439d565495d91e43afabcecbb6d03e9 Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Thu, 7 Feb 2019 10:39:29 +0100 Subject: [PATCH 08/71] C++: Add new query to new `experimental` suite This suite isn't referenced from anywhere yet, but it'll be included in a standard ODASA dist because the dist includes all files in the `c` and `cpp` directories. We can modify the nightly test jobs to include the experimental suite. --- cpp/config/suites/c/experimental | 1 + cpp/config/suites/cpp/experimental | 1 + 2 files changed, 2 insertions(+) create mode 100644 cpp/config/suites/c/experimental create mode 100644 cpp/config/suites/cpp/experimental diff --git a/cpp/config/suites/c/experimental b/cpp/config/suites/c/experimental new file mode 100644 index 000000000000..1842d710c6e1 --- /dev/null +++ b/cpp/config/suites/c/experimental @@ -0,0 +1 @@ ++ semmlecode-cpp-queries/Likely Bugs/RedundantNullCheckSimple.ql: /Correctness/Common Errors diff --git a/cpp/config/suites/cpp/experimental b/cpp/config/suites/cpp/experimental new file mode 100644 index 000000000000..1842d710c6e1 --- /dev/null +++ b/cpp/config/suites/cpp/experimental @@ -0,0 +1 @@ ++ semmlecode-cpp-queries/Likely Bugs/RedundantNullCheckSimple.ql: /Correctness/Common Errors From 9f2fdbbc1daa45e2e49200902a53f0ba5eede4fc Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Tue, 19 Feb 2019 17:12:33 +0100 Subject: [PATCH 09/71] C++: More tests for RedundantNullCheckSimple --- .../RedundantNullCheckSimple.cpp | 42 ++++++++++++++++++- .../RedundantNullCheckSimple.expected | 1 + 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.cpp b/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.cpp index d693e50ac598..57173a06cc8d 100644 --- a/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.cpp +++ b/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.cpp @@ -1,4 +1,4 @@ -void test1(int *p) { +void test_simple_bad(int *p) { int x; x = *p; if (p == nullptr) { // BAD @@ -6,7 +6,7 @@ void test1(int *p) { } } -void test2(int *p) { +void test_not_same_basic_block(int *p) { int x = *p; if (x > 100) return; @@ -31,3 +31,41 @@ bool check_curslist(ContainsIntPtr *cip) { // an AliasedDefinition instruction. return *cip->intPtr != nullptr; // GOOD } + +void test_no_single_dominator(int *p, bool b) { + int x; + if (b) { + x = *p; + } else { + x = *p; + } + if (p == nullptr) { // BAD [NOT DETECTED] + return; + } +} + +int test_postdominator_same_bb(int *p) { + int b = (p == nullptr); // BAD + // This dereference is a postdominator of the null check, meaning that all + // paths from the check to the function exit will pass through it. + return *p + b; +} + +int test_postdominator(int *p) { + int b = (p == nullptr); // BAD [NOT DETECTED] + + if (b) b++; // This line breaks up the basic block + + // This dereference is a postdominator of the null check, meaning that all + // paths from the check to the function exit will pass through it. + return *p + b; +} + +int test_inverted_logic(int *p) { + if (p == nullptr) { // BAD [NOT DETECTED] + // The check above should probably have been `!=` instead of `==`. + return *p; + } else { + return 0; + } +} diff --git a/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.expected b/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.expected index c0ece0bfa815..0fa4471ebee8 100644 --- a/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.expected +++ b/cpp/ql/test/query-tests/Likely Bugs/RedundantNullCheckSimple/RedundantNullCheckSimple.expected @@ -1,2 +1,3 @@ | RedundantNullCheckSimple.cpp:4:7:4:7 | Load: p | This null check is redundant because the value is $@ in any case | RedundantNullCheckSimple.cpp:3:7:3:8 | Load: * ... | dereferenced here | | RedundantNullCheckSimple.cpp:13:8:13:8 | Load: p | This null check is redundant because the value is $@ in any case | RedundantNullCheckSimple.cpp:10:11:10:12 | Load: * ... | dereferenced here | +| RedundantNullCheckSimple.cpp:48:12:48:12 | Load: p | This null check is redundant because the value is $@ in any case | RedundantNullCheckSimple.cpp:51:10:51:11 | Load: * ... | dereferenced here | From bfbf686d7be9c4e0c2aa7cd46ea2ad836ced3bc0 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Tue, 12 Feb 2019 15:14:12 +0100 Subject: [PATCH 10/71] JS: fixup changenote for js/unbound-event-handler-receiver --- change-notes/1.20/analysis-javascript.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/change-notes/1.20/analysis-javascript.md b/change-notes/1.20/analysis-javascript.md index 88b8ec66d495..f0d5435e98bd 100644 --- a/change-notes/1.20/analysis-javascript.md +++ b/change-notes/1.20/analysis-javascript.md @@ -26,7 +26,6 @@ | Incomplete URL substring sanitization | correctness, security, external/cwe/cwe-020 | Highlights URL sanitizers that are likely to be incomplete, indicating a violation of [CWE-020](https://cwe.mitre.org/data/definitions/20.html). Results shown on LGTM by default. | | Incorrect suffix check (`js/incorrect-suffix-check`) | correctness, security, external/cwe/cwe-020 | Highlights error-prone suffix checks based on `indexOf`, indicating a potential violation of [CWE-20](https://cwe.mitre.org/data/definitions/20.html). Results are shown on LGTM by default. | | Loop iteration skipped due to shifting (`js/loop-iteration-skipped-due-to-shifting`) | correctness | Highlights code that removes an element from an array while iterating over it, causing the loop to skip over some elements. Results are shown on LGTM by default. | -| Unbound event handler receiver (`js/unbound-event-handler-receiver`) | Fewer false positive results | Additional ways that class methods can be bound are recognized. | | Useless comparison test (`js/useless-comparison-test`) | correctness | Highlights code that is unreachable due to a numeric comparison that is always true or always false. Results are shown on LGTM by default. | ## Changes to existing queries @@ -39,6 +38,7 @@ | Insecure randomness | More results | This rule now flags insecure uses of `crypto.pseudoRandomBytes`. | | Reflected cross-site scripting | Fewer false-positive results. | This rule now recognizes custom sanitizers. | | Stored cross-site scripting | Fewer false-positive results. | This rule now recognizes custom sanitizers. | +| Unbound event handler receiver (`js/unbound-event-handler-receiver`) | Fewer false positive results | Additional ways that class methods can be bound are recognized. | | Uncontrolled data used in network request | More results | This rule now recognizes host values that are vulnerable to injection. | | Unused parameter | Fewer false-positive results | This rule no longer flags parameters with leading underscore. | | Unused variable, import, function or class | Fewer false-positive results | This rule now flags fewer variables that are implictly used by JSX elements, and no longer flags variables with leading underscore. | From 0cf2eaec5ee3e6c053f5b685c6f9d89020a1dbe2 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Tue, 12 Feb 2019 15:57:52 +0100 Subject: [PATCH 11/71] JS: introduce CapturedSource --- .../javascript/dataflow/CapturedNodes.qll | 74 +++++++++++++++ .../CapturedNodes/CapturedSource.expected | 25 +++++ .../CapturedNodes/CapturedSource.ql | 4 + .../CapturedSource_hasOwnProperty.expected | 22 +++++ .../CapturedSource_hasOwnProperty.ql | 6 ++ .../MethodCallTypeInference.expected | 12 +++ .../CapturedNodes/MethodCallTypeInference.ql | 4 + .../MethodCallTypeInferenceUsage.expected | 4 + .../MethodCallTypeInferenceUsage.ql | 5 + .../CapturedNodes/method-calls.js | 57 ++++++++++++ .../test/library-tests/CapturedNodes/tst.js | 93 +++++++++++++++++++ .../test/library-tests/CapturedNodes/tst.ts | 6 ++ 12 files changed, 312 insertions(+) create mode 100644 javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll create mode 100644 javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected create mode 100644 javascript/ql/test/library-tests/CapturedNodes/CapturedSource.ql create mode 100644 javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected create mode 100644 javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.ql create mode 100644 javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected create mode 100644 javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.ql create mode 100644 javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected create mode 100644 javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.ql create mode 100644 javascript/ql/test/library-tests/CapturedNodes/method-calls.js create mode 100644 javascript/ql/test/library-tests/CapturedNodes/tst.js create mode 100644 javascript/ql/test/library-tests/CapturedNodes/tst.ts diff --git a/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll b/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll new file mode 100644 index 000000000000..bcb9c44abe5a --- /dev/null +++ b/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll @@ -0,0 +1,74 @@ +/** + * Provides classes for the nodes that the dataflow library can reason about soundly. + */ + +import javascript + +/** + * Holds if the dataflow library can not track flow through `escape` due to `cause`. + */ +private predicate isEscape(DataFlow::Node escape, string cause) { + escape = any(DataFlow::InvokeNode invk).getAnArgument() and cause = "argument" + or + escape = any(DataFlow::FunctionNode fun).getAReturn() and cause = "return" + or + escape = any(ThrowStmt t).getExpr().flow() and cause = "throw" + or + escape = any(DataFlow::GlobalVariable v).getAnAssignedExpr().flow() and cause = "global" + or + escape = any(DataFlow::PropWrite write).getRhs() and cause = "heap" + or + escape = any(ExportDeclaration e).getSourceNode(_) and cause = "export" + or + any(WithStmt with).mayAffect(escape.asExpr()) and cause = "heap" +} + +private DataFlow::Node getAnEscape() { + isEscape(result, _) +} + +/** + * Holds if `n` can flow to a `this`-variable. + */ +private predicate exposedAsReceiver(DataFlow::SourceNode n) { + // pragmatic limitation: guarantee for object literals only + not n instanceof DataFlow::ObjectLiteralNode + or + exists(AbstractValue v | n.getAPropertyWrite().getRhs().analyze().getALocalValue() = v | + v.isIndefinite(_) or + exists(ThisExpr dis | dis.getBinder() = v.(AbstractCallable).getFunction()) + ) + or + n.flowsToExpr(any(FunctionBindExpr bind).getObject()) + or + // technically, the builtin prototypes could have a `this`-using function through which this node escapes, but we ignore that here + // (we also ignore `o['__' + 'proto__"] = ...`) + exists(n.getAPropertyWrite("__proto__")) + or + // could check the assigned value of all affected variables, but it is unlikely to matter in practice + exists(WithStmt with | n.flowsToExpr(with.getExpr())) +} + +/** + * A source for which the flow is entirely captured by the dataflow library. + * All uses of the node is represented by `this.flowsTo(_)` and friends. + */ +class CapturedSource extends DataFlow::SourceNode { + CapturedSource() { + // pragmatic limitation: object literals only + this instanceof DataFlow::ObjectLiteralNode and + not flowsTo(getAnEscape()) and + not exposedAsReceiver(this) + } + + predicate hasOwnProperty(string name) { + // the property is defined in the initializer, + any(DataFlow::PropWrite write).writes(this, name, _) and + // and it is never deleted + not exists(DeleteExpr del, DataFlow::PropRef ref | + del.getOperand().flow() = ref and + flowsTo(ref.getBase()) and + (ref.getPropertyName() = name or not exists(ref.getPropertyName())) + ) + } +} diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected new file mode 100644 index 000000000000..d78dd6e228ff --- /dev/null +++ b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected @@ -0,0 +1,25 @@ +| method-calls.js:2:11:7:2 | {\\n\\t\\tm1: ... }; }\\n\\t} | +| method-calls.js:11:23:11:24 | {} | +| method-calls.js:16:11:16:12 | {} | +| method-calls.js:23:11:23:17 | {m: f1} | +| method-calls.js:27:11:27:17 | {m: f2} | +| method-calls.js:32:12:32:18 | {m: f3} | +| method-calls.js:36:15:36:21 | {m: f2} | +| method-calls.js:42:16:42:28 | {m: () => 42} | +| method-calls.js:46:17:46:29 | {m: () => 42} | +| method-calls.js:50:16:50:28 | {m: () => 42} | +| method-calls.js:53:16:53:28 | {m: () => 42} | +| tst.js:3:18:3:19 | {} | +| tst.js:4:18:4:36 | { f: function(){} } | +| tst.js:5:18:5:34 | { [unknown]: 42 } | +| tst.js:38:18:38:26 | { p: 42 } | +| tst.js:42:18:42:33 | { p: 42, q: 42 } | +| tst.js:52:23:52:24 | {} | +| tst.js:56:20:56:35 | { p: 42, p: 42 } | +| tst.js:59:20:59:28 | { p: 42 } | +| tst.js:63:20:63:28 | { p: 42 } | +| tst.js:68:19:68:27 | { p: 42 } | +| tst.js:73:18:73:20 | { } | +| tst.js:76:18:76:26 | { p: 42 } | +| tst.js:77:18:77:28 | { p: true } | +| tst.js:82:20:82:21 | {} | diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.ql b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.ql new file mode 100644 index 000000000000..0731e04afee1 --- /dev/null +++ b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.ql @@ -0,0 +1,4 @@ +import javascript +import semmle.javascript.dataflow.CapturedNodes + +select any(CapturedSource n) diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected new file mode 100644 index 000000000000..76e494cf1ad0 --- /dev/null +++ b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected @@ -0,0 +1,22 @@ +| method-calls.js:2:11:7:2 | {\\n\\t\\tm1: ... }; }\\n\\t} | m1 | +| method-calls.js:2:11:7:2 | {\\n\\t\\tm1: ... }; }\\n\\t} | m2 | +| method-calls.js:2:11:7:2 | {\\n\\t\\tm1: ... }; }\\n\\t} | m3 | +| method-calls.js:2:11:7:2 | {\\n\\t\\tm1: ... }; }\\n\\t} | m4 | +| method-calls.js:23:11:23:17 | {m: f1} | m | +| method-calls.js:27:11:27:17 | {m: f2} | m | +| method-calls.js:32:12:32:18 | {m: f3} | m | +| method-calls.js:36:15:36:21 | {m: f2} | m | +| method-calls.js:42:16:42:28 | {m: () => 42} | m | +| method-calls.js:46:17:46:29 | {m: () => 42} | m | +| method-calls.js:50:16:50:28 | {m: () => 42} | m | +| method-calls.js:53:16:53:28 | {m: () => 42} | m | +| tst.js:4:18:4:36 | { f: function(){} } | f | +| tst.js:38:18:38:26 | { p: 42 } | p | +| tst.js:42:18:42:33 | { p: 42, q: 42 } | p | +| tst.js:42:18:42:33 | { p: 42, q: 42 } | q | +| tst.js:56:20:56:35 | { p: 42, p: 42 } | p | +| tst.js:59:20:59:28 | { p: 42 } | p | +| tst.js:63:20:63:28 | { p: 42 } | p | +| tst.js:68:19:68:27 | { p: 42 } | p | +| tst.js:76:18:76:26 | { p: 42 } | p | +| tst.js:77:18:77:28 | { p: true } | p | diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.ql b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.ql new file mode 100644 index 000000000000..14911b329daa --- /dev/null +++ b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.ql @@ -0,0 +1,6 @@ +import javascript +import semmle.javascript.dataflow.CapturedNodes + +from CapturedSource src, string name +where src.hasOwnProperty(name) +select src, name diff --git a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected new file mode 100644 index 000000000000..4c11147ddead --- /dev/null +++ b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected @@ -0,0 +1,12 @@ +| method-calls.js:8:2:8:8 | o1.m1() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:9:2:9:8 | o1.m2() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:12:2:12:7 | o.m3() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:19:2:19:7 | o2.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:23:11:23:21 | {m: f1}.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:28:11:28:16 | o2.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:32:11:32:23 | ({m: f3}).m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:37:11:37:16 | o4.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:43:12:43:16 | o.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:47:12:47:16 | o.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:51:12:51:16 | o.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:54:12:54:16 | o.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | diff --git a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.ql b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.ql new file mode 100644 index 000000000000..b5224bed0676 --- /dev/null +++ b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.ql @@ -0,0 +1,4 @@ +import javascript + +from DataFlow::MethodCallNode call +select call, call.analyze().ppTypes() diff --git a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected new file mode 100644 index 000000000000..8d3ed2834a6f --- /dev/null +++ b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected @@ -0,0 +1,4 @@ +| method-calls.js:23:11:23:21 | {m: f1}.m() | method-calls.js:24:2:24:3 | v1 | file://:0:0:0:0 | indefinite value (call) | +| method-calls.js:28:11:28:16 | o2.m() | method-calls.js:29:2:29:3 | v2 | file://:0:0:0:0 | indefinite value (call) | +| method-calls.js:32:11:32:23 | ({m: f3}).m() | method-calls.js:33:2:33:3 | v3 | file://:0:0:0:0 | indefinite value (call) | +| method-calls.js:37:11:37:16 | o4.m() | method-calls.js:38:2:38:3 | v4 | file://:0:0:0:0 | indefinite value (call) | diff --git a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.ql b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.ql new file mode 100644 index 000000000000..e35d82a37370 --- /dev/null +++ b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.ql @@ -0,0 +1,5 @@ +import javascript + +from DataFlow::MethodCallNode call, DataFlow::Node use +where call.flowsTo(use) and use != call and not exists(use.getASuccessor()) +select call, use, use.analyze().getAValue() \ No newline at end of file diff --git a/javascript/ql/test/library-tests/CapturedNodes/method-calls.js b/javascript/ql/test/library-tests/CapturedNodes/method-calls.js new file mode 100644 index 000000000000..f17f72e58962 --- /dev/null +++ b/javascript/ql/test/library-tests/CapturedNodes/method-calls.js @@ -0,0 +1,57 @@ +(function() { + var o1 = { + m1: function(){ return {}; }, + m2: function(){ return {}; }, + m3: function(){ return {}; }, + m4: function(){ return {}; } + }; + o1.m1(); + o1.m2(); + unknown(o1.m2); + var o = unknown? o1: {}; + o.m3(); // NOT supported + var m4 = o.m4; + m4(); + + var o2 = {}; + o2.m = function() { return {}; }; + o2[unknown] = function() { return true; }; // could be __proto__ + o2.m(); +}); +(function(){ + function f1(){return {};} + var v1 = {m: f1}.m(); + v1 === true; + + function f2(){return {};} + var o2 = {m: f2}; + var v2 = o2.m(); + v2 === true; + + function f3(){return {};} + var v3 = ({m: f3}).m(); + v3 === true; + + function f4(){return {};} + var { o4 } = {m: f2}; + var v4 = o4.m(); + v4 === true; +}); + +(function(){ + (function(o = {m: () => 42}){ + var v1 = o.m(); + })(unknown); + + function f(o = {m: () => 42}){ + var v2 = o.m(); + }; + f(unknown); + (function(o = {m: () => 42}){ + var v3 = o.m(); + })({m: unknown}); + (function(o = {m: () => 42}){ + var v4 = o.m(); + })({m: () => true}); + +}); diff --git a/javascript/ql/test/library-tests/CapturedNodes/tst.js b/javascript/ql/test/library-tests/CapturedNodes/tst.js new file mode 100644 index 000000000000..dee138c0f270 --- /dev/null +++ b/javascript/ql/test/library-tests/CapturedNodes/tst.js @@ -0,0 +1,93 @@ +(function capturedSource(){ + + let captured1 = {}; + let captured2 = { f: function(){} }; + let captured3 = { [unknown]: 42 }; + + unknown({}); + + function known(){} + known({}); + + function known_escaping(e){unknown(e)} + known_escaping({}); + + (function(){return {}}); + + (function(){throw {}}); + + global = {}; + + this.p = {}; + + let local_in_with; + with (unknown) { + local_in_with = {}; + } + + with({}){} + + ({ m: function(){ this; } }); + ({ m: unknown }); + let indirectlyUnknown = unknown? unknown: function(){}; + ({ m: indirectlyUnknown }); +}); + +(function capturedProperty(){ + + let captured1 = { p: 42 }; + captured1.p; + captured1.p; + + let captured2 = { p: 42, q: 42 }; + captured2.p; + captured2.p; + captured2.q = 42; + captured2 = 42; + + let nonObject = function(){} + nonObject.p = 42; + nonObject.p; + + let nonInitializer = {}; + nonInitializer.p = 42; + nonInitializer.p; + + let overridden1 = { p: 42, p: 42 }; + overridden1.p; + + let overridden2 = { p: 42 }; + overridden2.p = 42; + overridden2.p; + + let overridden3 = { p: 42 }; + overridden3[x] = 42; + overridden3.p; + + function f(o) { + let captured3 = { p: 42 }; + o = o || captured3; + o.p; + } + + let captured4 = { }; + captured4.p; + + let captured5 = { p: 42 }, + captured6 = { p: true }; + (unknown? captured5: captured6).p; // could support this with a bit of extra work + + (function(semiCaptured7){ + if(unknown) + semiCaptured7 = {}; + semiCaptured7.p = 42; + }); + +}); + +(function (){ + let bound = {}; + bound::unknown(); +}); + +// semmle-extractor-options: --experimental diff --git a/javascript/ql/test/library-tests/CapturedNodes/tst.ts b/javascript/ql/test/library-tests/CapturedNodes/tst.ts new file mode 100644 index 000000000000..1f2794c0e5e1 --- /dev/null +++ b/javascript/ql/test/library-tests/CapturedNodes/tst.ts @@ -0,0 +1,6 @@ +class C { + constructor( + private readonly F: { timeout: number } = { timeout: 1500 } + ) { + } +} From 91dccc3356df024b7f90a5d6abca77c79e75d363 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Tue, 12 Feb 2019 15:02:03 +0100 Subject: [PATCH 12/71] JS: add query js/unused-property --- .../suites/javascript/maintainability-more | 1 + .../ql/src/Declarations/UnusedProperty.qhelp | 34 ++++++++ .../ql/src/Declarations/UnusedProperty.ql | 74 +++++++++++++++++ .../ql/src/Declarations/UnusedVariable.ql | 18 +--- .../ql/src/Declarations/UnusedVariable.qll | 22 +++++ .../Declarations/examples/UnusedProperty.js | 8 ++ .../ql/src/Expressions/ExprHasNoEffect.ql | 35 +------- .../ql/src/Expressions/ExprHasNoEffect.qll | 39 +++++++++ .../UnusedProperty/UnusedProperty.expected | 9 ++ .../UnusedProperty/UnusedProperty.qlref | 1 + .../Declarations/UnusedProperty/tst.js | 83 +++++++++++++++++++ .../Declarations/UnusedProperty/tst.ts | 28 +++++++ 12 files changed, 301 insertions(+), 51 deletions(-) create mode 100644 javascript/ql/src/Declarations/UnusedProperty.qhelp create mode 100644 javascript/ql/src/Declarations/UnusedProperty.ql create mode 100644 javascript/ql/src/Declarations/UnusedVariable.qll create mode 100644 javascript/ql/src/Declarations/examples/UnusedProperty.js create mode 100644 javascript/ql/src/Expressions/ExprHasNoEffect.qll create mode 100644 javascript/ql/test/query-tests/Declarations/UnusedProperty/UnusedProperty.expected create mode 100644 javascript/ql/test/query-tests/Declarations/UnusedProperty/UnusedProperty.qlref create mode 100644 javascript/ql/test/query-tests/Declarations/UnusedProperty/tst.js create mode 100644 javascript/ql/test/query-tests/Declarations/UnusedProperty/tst.ts diff --git a/javascript/config/suites/javascript/maintainability-more b/javascript/config/suites/javascript/maintainability-more index f887a26e1797..0f09b9ee4e6b 100644 --- a/javascript/config/suites/javascript/maintainability-more +++ b/javascript/config/suites/javascript/maintainability-more @@ -5,6 +5,7 @@ + semmlecode-javascript-queries/Declarations/DeadStoreOfProperty.ql: /Maintainability/Declarations + semmlecode-javascript-queries/Declarations/DuplicateVarDecl.ql: /Maintainability/Declarations + semmlecode-javascript-queries/Declarations/UnusedParameter.ql: /Maintainability/Declarations ++ semmlecode-javascript-queries/Declarations/UnusedProperty.ql: /Maintainability/Declarations + semmlecode-javascript-queries/Declarations/UnusedVariable.ql: /Maintainability/Declarations + semmlecode-javascript-queries/Expressions/UnneededDefensiveProgramming.ql: /Maintainability/Expressions + semmlecode-javascript-queries/LanguageFeatures/ArgumentsCallerCallee.ql: /Maintainability/Language Features diff --git a/javascript/ql/src/Declarations/UnusedProperty.qhelp b/javascript/ql/src/Declarations/UnusedProperty.qhelp new file mode 100644 index 000000000000..6da2b7d16f40 --- /dev/null +++ b/javascript/ql/src/Declarations/UnusedProperty.qhelp @@ -0,0 +1,34 @@ + + + +

    + Unused object properties make code harder to maintain and use. Clients that are unaware that a + property is unused may perform nontrivial computations to compute a value that is ultimately + unused. +

    + +     + +

    Remove the unused property.

    + +     + + +

    + In this code, the function f initializes a property prop_a with a + call to the function expensiveComputation, but later on this property is never read. + Removing prop would improve code quality and performance. +

    + + + +     + + +
  • Coding Horror: Code Smells.
    • + + +     +     diff --git a/javascript/ql/src/Declarations/UnusedProperty.ql b/javascript/ql/src/Declarations/UnusedProperty.ql new file mode 100644 index 000000000000..b364a2b21483 --- /dev/null +++ b/javascript/ql/src/Declarations/UnusedProperty.ql @@ -0,0 +1,74 @@ +/** + * @name Unused property + * @description Unused properties may be a symptom of a bug and should be examined carefully. + * @kind problem + * @problem.severity recommendation + * @id js/unused-property + * @tags maintainability + * @precision high + */ + +import javascript +import semmle.javascript.dataflow.CapturedNodes +import UnusedVariable +import UnusedParameter +import Expressions.ExprHasNoEffect + +predicate hasUnknownPropertyRead(CapturedSource obj) { + // dynamic reads + exists(DataFlow::PropRead r | obj.getAPropertyRead() = r | not exists(r.getPropertyName())) + or + // reflective reads + obj.flowsToExpr(any(EnhancedForLoop l).getIterationDomain()) + or + obj.flowsToExpr(any(InExpr l).getRightOperand()) + or + obj.flowsToExpr(any(SpreadElement e).getOperand()) + or + exists(obj.getAPropertyRead("hasOwnProperty")) + or + exists(obj.getAPropertyRead("propertyIsEnumerable")) +} + +predicate flowsToTypeRestrictedExpression(CapturedSource n) { + exists (Expr restricted, TypeExpr type | + n.flowsToExpr(restricted) and + not type.isAny() | + exists (TypeAssertion assertion | + type = assertion.getTypeAnnotation() and + restricted = assertion.getExpression() + ) + or + exists (BindingPattern v | + type = v.getTypeAnnotation() and + restricted = v.getAVariable().getAnAssignedExpr() + ) + // no need to reason about writes to typed fields, captured nodes do not reach them + ) +} + +from DataFlow::PropWrite w, CapturedSource n, string name +where + w = n.getAPropertyWrite(name) and + not exists(n.getAPropertyRead(name)) and + not w.getBase().analyze().getAValue() != n.analyze().getAValue() and + not hasUnknownPropertyRead(n) and + // avoid reporting if the definition is unreachable + w.getAstNode().getFirstControlFlowNode().getBasicBlock() instanceof ReachableBasicBlock and + // avoid implicitly read properties + not ( + name = "toString" or + name = "valueOf" or + name.matches("@@%") // @@iterator, for example + ) and + // avoid flagging properties that a type system requires + not flowsToTypeRestrictedExpression(n) and + // flagged by js/unused-local-variable + not exists(UnusedLocal l | l.getAnAssignedExpr().getUnderlyingValue().flow() = n) and + // flagged by js/unused-parameter + not exists(Parameter p | isAnAccidentallyUnusedParameter(p) | + p.getDefault().getUnderlyingValue().flow() = n + ) and + // flagged by js/useless-expression + not inVoidContext(n.asExpr()) +select w, "Unused property " + name + "." diff --git a/javascript/ql/src/Declarations/UnusedVariable.ql b/javascript/ql/src/Declarations/UnusedVariable.ql index aa361582c10c..10eb914701eb 100644 --- a/javascript/ql/src/Declarations/UnusedVariable.ql +++ b/javascript/ql/src/Declarations/UnusedVariable.ql @@ -10,23 +10,7 @@ */ import javascript - -/** - * A local variable that is neither used nor exported, and is not a parameter - * or a function name. - */ -class UnusedLocal extends LocalVariable { - UnusedLocal() { - not exists(getAnAccess()) and - not exists(Parameter p | this = p.getAVariable()) and - not exists(FunctionExpr fe | this = fe.getVariable()) and - not exists(ClassExpr ce | this = ce.getVariable()) and - not exists(ExportDeclaration ed | ed.exportsAs(this, _)) and - not exists(LocalVarTypeAccess type | type.getVariable() = this) and - // common convention: variables with leading underscore are intentionally unused - getName().charAt(0) != "_" - } -} +import UnusedVariable /** * Holds if `v` is mentioned in a JSDoc comment in the same file, and that file diff --git a/javascript/ql/src/Declarations/UnusedVariable.qll b/javascript/ql/src/Declarations/UnusedVariable.qll new file mode 100644 index 000000000000..98d69b80938c --- /dev/null +++ b/javascript/ql/src/Declarations/UnusedVariable.qll @@ -0,0 +1,22 @@ +/** + * This library contains parts of the 'js/unused-local-variable' query implementation. + */ + +import javascript + +/** + * A local variable that is neither used nor exported, and is not a parameter + * or a function name. + */ +class UnusedLocal extends LocalVariable { + UnusedLocal() { + not exists(getAnAccess()) and + not exists(Parameter p | this = p.getAVariable()) and + not exists(FunctionExpr fe | this = fe.getVariable()) and + not exists(ClassExpr ce | this = ce.getVariable()) and + not exists(ExportDeclaration ed | ed.exportsAs(this, _)) and + not exists(LocalVarTypeAccess type | type.getVariable() = this) and + // common convention: variables with leading underscore are intentionally unused + getName().charAt(0) != "_" + } +} diff --git a/javascript/ql/src/Declarations/examples/UnusedProperty.js b/javascript/ql/src/Declarations/examples/UnusedProperty.js new file mode 100644 index 000000000000..3d1441d430b2 --- /dev/null +++ b/javascript/ql/src/Declarations/examples/UnusedProperty.js @@ -0,0 +1,8 @@ +function f() { + var o = { + prop_a: expensiveComputation(), + prop_b: anotherComputation() + }; + + return o.prop_b; +} diff --git a/javascript/ql/src/Expressions/ExprHasNoEffect.ql b/javascript/ql/src/Expressions/ExprHasNoEffect.ql index c2dbe038963d..3ceda6fa29bc 100644 --- a/javascript/ql/src/Expressions/ExprHasNoEffect.ql +++ b/javascript/ql/src/Expressions/ExprHasNoEffect.ql @@ -16,40 +16,7 @@ import javascript import DOMProperties import semmle.javascript.frameworks.xUnit import semmle.javascript.RestrictedLocations - -/** - * Holds if `e` appears in a syntactic context where its value is discarded. - */ -predicate inVoidContext(Expr e) { - exists(ExprStmt parent | - // e is a toplevel expression in an expression statement - parent = e.getParent() and - // but it isn't an HTML attribute or a configuration object - not exists(TopLevel tl | tl = parent.getParent() | - tl instanceof CodeInAttribute - or - // if the toplevel in its entirety is of the form `({ ... })`, - // it is probably a configuration object (e.g., a require.js build configuration) - tl.getNumChildStmt() = 1 and e.stripParens() instanceof ObjectExpr - ) - ) - or - exists(SeqExpr seq, int i, int n | - e = seq.getOperand(i) and - n = seq.getNumOperands() - | - i < n - 1 or inVoidContext(seq) - ) - or - exists(ForStmt stmt | e = stmt.getUpdate()) - or - exists(ForStmt stmt | e = stmt.getInit() | - // Allow the pattern `for(i; i < 10; i++)` - not e instanceof VarAccess - ) - or - exists(LogicalBinaryExpr logical | e = logical.getRightOperand() and inVoidContext(logical)) -} +import ExprHasNoEffect /** * Holds if `e` is of the form `x;` or `e.p;` and has a JSDoc comment containing a tag. diff --git a/javascript/ql/src/Expressions/ExprHasNoEffect.qll b/javascript/ql/src/Expressions/ExprHasNoEffect.qll new file mode 100644 index 000000000000..3b6f4703ad6e --- /dev/null +++ b/javascript/ql/src/Expressions/ExprHasNoEffect.qll @@ -0,0 +1,39 @@ +/** + * This library contains parts of the 'js/useless-expression' query implementation. + */ + +import javascript + +/** + * Holds if `e` appears in a syntactic context where its value is discarded. + */ +predicate inVoidContext(Expr e) { + exists(ExprStmt parent | + // e is a toplevel expression in an expression statement + parent = e.getParent() and + // but it isn't an HTML attribute or a configuration object + not exists(TopLevel tl | tl = parent.getParent() | + tl instanceof CodeInAttribute + or + // if the toplevel in its entirety is of the form `({ ... })`, + // it is probably a configuration object (e.g., a require.js build configuration) + tl.getNumChildStmt() = 1 and e.stripParens() instanceof ObjectExpr + ) + ) + or + exists(SeqExpr seq, int i, int n | + e = seq.getOperand(i) and + n = seq.getNumOperands() + | + i < n - 1 or inVoidContext(seq) + ) + or + exists(ForStmt stmt | e = stmt.getUpdate()) + or + exists(ForStmt stmt | e = stmt.getInit() | + // Allow the pattern `for(i; i < 10; i++)` + not e instanceof VarAccess + ) + or + exists(LogicalBinaryExpr logical | e = logical.getRightOperand() and inVoidContext(logical)) +} diff --git a/javascript/ql/test/query-tests/Declarations/UnusedProperty/UnusedProperty.expected b/javascript/ql/test/query-tests/Declarations/UnusedProperty/UnusedProperty.expected new file mode 100644 index 000000000000..f976725b5592 --- /dev/null +++ b/javascript/ql/test/query-tests/Declarations/UnusedProperty/UnusedProperty.expected @@ -0,0 +1,9 @@ +| tst.js:4:9:4:19 | unused1: 42 | Unused property unused1. | +| tst.js:19:5:19:15 | unused9: 42 | Unused property unused9. | +| tst.js:26:13:26:24 | unused11: 42 | Unused property unused11. | +| tst.js:31:13:31:35 | used12_ ... lly: 42 | Unused property used12_butNotReally. | +| tst.js:32:13:32:24 | unused12: 42 | Unused property unused12. | +| tst.js:52:3:52:14 | unused14: 42 | Unused property unused14. | +| tst.js:54:2:54:20 | captured14.unused14 | Unused property unused14. | +| tst.js:55:2:55:20 | captured14.unused14 | Unused property unused14. | +| tst.ts:24:21:24:25 | p: 42 | Unused property p. | diff --git a/javascript/ql/test/query-tests/Declarations/UnusedProperty/UnusedProperty.qlref b/javascript/ql/test/query-tests/Declarations/UnusedProperty/UnusedProperty.qlref new file mode 100644 index 000000000000..9583241c2f0d --- /dev/null +++ b/javascript/ql/test/query-tests/Declarations/UnusedProperty/UnusedProperty.qlref @@ -0,0 +1 @@ +Declarations/UnusedProperty.ql diff --git a/javascript/ql/test/query-tests/Declarations/UnusedProperty/tst.js b/javascript/ql/test/query-tests/Declarations/UnusedProperty/tst.js new file mode 100644 index 000000000000..26cd1a110bba --- /dev/null +++ b/javascript/ql/test/query-tests/Declarations/UnusedProperty/tst.js @@ -0,0 +1,83 @@ +(function(){ + var captured1 = { + used1: 42, + unused1: 42 + }; + captured1.used1; + + var unused2 = { + unused2a: 42, + unused2b: 42 + }; + + for (x.p in { used3: 42 }); + for (x.p of { used4: 42 }); + 42 in { used5: 42 }; + f(...{used6: 42}); + [...{used7: 42}]; + ({...{used8: 42}}); + ({ unused9: 42 }) + ""; + ({ used10: 42 }).hasOwnProperty; + ({ used10: 42 }).propertyIsEnumerable; + + (function(){ + var captured11 = { + used11: 42, + unused11: 42 + }; + captured11.used11; + + var captured12 = { + used12_butNotReally: 42, + unused12: 42 + }; + + throw x; + + captured12.used12_butNotReally; + + var captured13 = { + used13: 42, + unused13: 42 + }; + captured13.used13; + }); + (function(options){ + if(unknown) + options = {}; + options.output = 42; + }); + + var captured14 = { + unused14: 42 + }; + captured14.unused14 = 42; + captured14.unused14 = 42; + + + var captured15 = { + semiUnused15: 42 + }; + captured15.semiUnused15 = 42; + captured15.semiUnused15; +}); +(function(unusedParam = {unusedProp: 42}){ + +}); +(function(){ + var unusedObj = { + unusedProp: 42 + }; +}); +(function(){ + var unusedSpecials = { + toString: function(){}, + valueOf: function(){}, + '@@iterator': function(){} + }; + unusedSpecials.foo; +}); + +(function(){ + ({ unusedProp: 42 }, 42); +}); diff --git a/javascript/ql/test/query-tests/Declarations/UnusedProperty/tst.ts b/javascript/ql/test/query-tests/Declarations/UnusedProperty/tst.ts new file mode 100644 index 000000000000..7ad7c508df8d --- /dev/null +++ b/javascript/ql/test/query-tests/Declarations/UnusedProperty/tst.ts @@ -0,0 +1,28 @@ +(function(){ + var o1: { p: int, q: int } = { p: 42, q: 42 }; + o1.q; + + var o2 = <{ p: int, q: int }>{ p: 42, q: 42 }; + o2.q; + + var o3: { p: int, q: int } = f(); + o3 = o3 || { p: 42, q: 42 }; + o3.q; + +}); + +class C { + private o: { p: int, q: int }; + + constructor() { + this.o = { p: 42, q: 42 }; + this.o.q; + } +} + +(function(){ + var o1: any = { p: 42, q: 42 }; + o1.q; + var o2: any = { p: 42, q: 42 }; + var o3: { p: int, q: int } = o2; +}) From 8af501d4d53771c12195ce89e39068226d1199eb Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Tue, 12 Feb 2019 15:07:28 +0100 Subject: [PATCH 13/71] JS: avoid double reporting dead code with js/unused-variable --- javascript/ql/src/Declarations/UnusedVariable.ql | 8 ++++++-- .../test/query-tests/Declarations/UnusedVariable/dead.js | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 javascript/ql/test/query-tests/Declarations/UnusedVariable/dead.js diff --git a/javascript/ql/src/Declarations/UnusedVariable.ql b/javascript/ql/src/Declarations/UnusedVariable.ql index 10eb914701eb..2c86549c4ec4 100644 --- a/javascript/ql/src/Declarations/UnusedVariable.ql +++ b/javascript/ql/src/Declarations/UnusedVariable.ql @@ -190,6 +190,10 @@ predicate unusedImports(ImportVarDeclProvider provider, string msg) { from ASTNode sel, string msg where - unusedNonImports(sel, msg) or - unusedImports(sel, msg) + ( + unusedNonImports(sel, msg) or + unusedImports(sel, msg) + ) and + // avoid reporting if the definition is unreachable + sel.getFirstControlFlowNode().getBasicBlock() instanceof ReachableBasicBlock select sel, msg diff --git a/javascript/ql/test/query-tests/Declarations/UnusedVariable/dead.js b/javascript/ql/test/query-tests/Declarations/UnusedVariable/dead.js new file mode 100644 index 000000000000..10fcd168991e --- /dev/null +++ b/javascript/ql/test/query-tests/Declarations/UnusedVariable/dead.js @@ -0,0 +1,4 @@ +(function(){ + throw 42; + var x = 42; +}); From c84d89872710441d07862cf6a20c30a12cb54522 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Tue, 12 Feb 2019 15:15:27 +0100 Subject: [PATCH 14/71] JS: change notes for js/unused-property and js/unused-variable --- change-notes/1.20/analysis-javascript.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/change-notes/1.20/analysis-javascript.md b/change-notes/1.20/analysis-javascript.md index f0d5435e98bd..20c5ed006a3b 100644 --- a/change-notes/1.20/analysis-javascript.md +++ b/change-notes/1.20/analysis-javascript.md @@ -26,6 +26,7 @@ | Incomplete URL substring sanitization | correctness, security, external/cwe/cwe-020 | Highlights URL sanitizers that are likely to be incomplete, indicating a violation of [CWE-020](https://cwe.mitre.org/data/definitions/20.html). Results shown on LGTM by default. | | Incorrect suffix check (`js/incorrect-suffix-check`) | correctness, security, external/cwe/cwe-020 | Highlights error-prone suffix checks based on `indexOf`, indicating a potential violation of [CWE-20](https://cwe.mitre.org/data/definitions/20.html). Results are shown on LGTM by default. | | Loop iteration skipped due to shifting (`js/loop-iteration-skipped-due-to-shifting`) | correctness | Highlights code that removes an element from an array while iterating over it, causing the loop to skip over some elements. Results are shown on LGTM by default. | +| Unused property (`js/unused-property`) | maintainability | Highlights properties that are unused. Results are shown on LGTM by default. | | Useless comparison test (`js/useless-comparison-test`) | correctness | Highlights code that is unreachable due to a numeric comparison that is always true or always false. Results are shown on LGTM by default. | ## Changes to existing queries @@ -41,7 +42,7 @@ | Unbound event handler receiver (`js/unbound-event-handler-receiver`) | Fewer false positive results | Additional ways that class methods can be bound are recognized. | | Uncontrolled data used in network request | More results | This rule now recognizes host values that are vulnerable to injection. | | Unused parameter | Fewer false-positive results | This rule no longer flags parameters with leading underscore. | -| Unused variable, import, function or class | Fewer false-positive results | This rule now flags fewer variables that are implictly used by JSX elements, and no longer flags variables with leading underscore. | +| Unused variable, import, function or class | Fewer false-positive results | This rule now flags fewer variables that are implictly used by JSX elements, no longer flags variables with leading underscore, and no longer flags variables in dead code. | | Uncontrolled data used in path expression | Fewer false-positive results | This rule now recognizes the Express `root` option, which prevents path traversal. | | Unneeded defensive code | More true-positive results, fewer false-positive results. | This rule now recognizes additional defensive code patterns. | | Useless conditional | Fewer results | Additional defensive coding patterns are now ignored. | From bdd8691e65d236c0657455eae8ac8c055b3e868a Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Tue, 12 Feb 2019 13:06:24 +0100 Subject: [PATCH 15/71] JS: add type inference for the return value of captured method calls --- .../internal/InterProceduralTypeInference.qll | 46 +++++++++++++++++++ .../MethodCallTypeInference.expected | 10 ++-- .../MethodCallTypeInferenceUsage.expected | 6 +-- .../CallWithAnalyzedReturnFlow.expected | 1 + .../InvokeNodeValue.expected | 2 +- 5 files changed, 56 insertions(+), 9 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll b/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll index 459cfe7c2bae..fcb3e00fb255 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll @@ -7,6 +7,8 @@ import javascript import AbstractValuesImpl +import semmle.javascript.dataflow.CapturedNodes + /** * Flow analysis for `this` expressions inside functions. */ @@ -230,3 +232,47 @@ private class TypeInferredCalleeWithAnalyzedReturnFlow extends CallWithNonLocalA override AnalyzedFunction getACallee() { result = fun } } + +/** + * Holds if `call` uses `receiver` as its only receiver value. + */ +pragma[noinline] +private predicate hasDefiniteReceiver( + DataFlow::MethodCallNode call, CapturedSource receiver +) { + call = receiver.getAMethodCall() and + exists (DataFlow::AnalyzedNode receiverNode, AbstractValue abstractCapturedReceiver | + receiverNode = call.getReceiver() and + not receiverNode.getALocalValue().isIndefinite(_) and + abstractCapturedReceiver = receiver.analyze().getALocalValue() and + forall(DataFlow::AbstractValue v | + receiverNode.getALocalValue() = v | + v = abstractCapturedReceiver + ) + ) +} + +/** + * Enables inter-procedural type inference for the return value of a + * method call to a flow-insensitively type-inferred callee. + */ +class TypeInferredMethodWithAnalyzedReturnFlow extends CallWithNonLocalAnalyzedReturnFlow { + DataFlow::FunctionNode fun; + + TypeInferredMethodWithAnalyzedReturnFlow() { + exists(CapturedSource s, DataFlow::PropWrite w, string name | + this.(DataFlow::MethodCallNode).getMethodName() = name and + s.hasOwnProperty(name) and + hasDefiniteReceiver(this, s) and + w = s.getAPropertyWrite() and + fun.flowsTo(w.getRhs()) and + ( + not exists(w.getPropertyName()) + or + w.getPropertyName() = name + ) + ) + } + + override AnalyzedFunction getACallee() { result = fun } +} \ No newline at end of file diff --git a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected index 4c11147ddead..1192bcdb1275 100644 --- a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected +++ b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected @@ -1,10 +1,10 @@ -| method-calls.js:8:2:8:8 | o1.m1() | boolean, class, date, function, null, number, object, regular expression,string or undefined | -| method-calls.js:9:2:9:8 | o1.m2() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:8:2:8:8 | o1.m1() | object | +| method-calls.js:9:2:9:8 | o1.m2() | object | | method-calls.js:12:2:12:7 | o.m3() | boolean, class, date, function, null, number, object, regular expression,string or undefined | | method-calls.js:19:2:19:7 | o2.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | -| method-calls.js:23:11:23:21 | {m: f1}.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | -| method-calls.js:28:11:28:16 | o2.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | -| method-calls.js:32:11:32:23 | ({m: f3}).m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:23:11:23:21 | {m: f1}.m() | object | +| method-calls.js:28:11:28:16 | o2.m() | object | +| method-calls.js:32:11:32:23 | ({m: f3}).m() | object | | method-calls.js:37:11:37:16 | o4.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | | method-calls.js:43:12:43:16 | o.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | | method-calls.js:47:12:47:16 | o.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | diff --git a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected index 8d3ed2834a6f..7c393230c8bc 100644 --- a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected +++ b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected @@ -1,4 +1,4 @@ -| method-calls.js:23:11:23:21 | {m: f1}.m() | method-calls.js:24:2:24:3 | v1 | file://:0:0:0:0 | indefinite value (call) | -| method-calls.js:28:11:28:16 | o2.m() | method-calls.js:29:2:29:3 | v2 | file://:0:0:0:0 | indefinite value (call) | -| method-calls.js:32:11:32:23 | ({m: f3}).m() | method-calls.js:33:2:33:3 | v3 | file://:0:0:0:0 | indefinite value (call) | +| method-calls.js:23:11:23:21 | {m: f1}.m() | method-calls.js:24:2:24:3 | v1 | method-calls.js:22:23:22:24 | object literal | +| method-calls.js:28:11:28:16 | o2.m() | method-calls.js:29:2:29:3 | v2 | method-calls.js:26:23:26:24 | object literal | +| method-calls.js:32:11:32:23 | ({m: f3}).m() | method-calls.js:33:2:33:3 | v3 | method-calls.js:31:23:31:24 | object literal | | method-calls.js:37:11:37:16 | o4.m() | method-calls.js:38:2:38:3 | v4 | file://:0:0:0:0 | indefinite value (call) | diff --git a/javascript/ql/test/library-tests/TypeInference/CallWithAnalyzedReturnFlow/CallWithAnalyzedReturnFlow.expected b/javascript/ql/test/library-tests/TypeInference/CallWithAnalyzedReturnFlow/CallWithAnalyzedReturnFlow.expected index ab6686a25fff..650ca382a063 100644 --- a/javascript/ql/test/library-tests/TypeInference/CallWithAnalyzedReturnFlow/CallWithAnalyzedReturnFlow.expected +++ b/javascript/ql/test/library-tests/TypeInference/CallWithAnalyzedReturnFlow/CallWithAnalyzedReturnFlow.expected @@ -7,6 +7,7 @@ | tst.js:11:5:11:8 | f1() | file://:0:0:0:0 | undefined | | tst.js:14:5:14:8 | f2() | file://:0:0:0:0 | undefined | | tst.js:15:5:15:8 | f2() | file://:0:0:0:0 | undefined | +| tst.js:18:5:18:11 | o1.f3() | file://:0:0:0:0 | undefined | | tst.js:23:5:23:8 | f4() | tst.js:21:16:21:30 | function f5 | | tst.js:23:5:23:10 | f4()() | file://:0:0:0:0 | undefined | | tst.js:30:5:30:8 | f6() | tst.js:26:16:28:9 | function f7 | diff --git a/javascript/ql/test/library-tests/TypeInference/CallWithAnalyzedReturnFlow/InvokeNodeValue.expected b/javascript/ql/test/library-tests/TypeInference/CallWithAnalyzedReturnFlow/InvokeNodeValue.expected index 85b74fa0d7ed..a3a83471adab 100644 --- a/javascript/ql/test/library-tests/TypeInference/CallWithAnalyzedReturnFlow/InvokeNodeValue.expected +++ b/javascript/ql/test/library-tests/TypeInference/CallWithAnalyzedReturnFlow/InvokeNodeValue.expected @@ -11,7 +11,7 @@ | tst.js:11:5:11:8 | f1() | file://:0:0:0:0 | undefined | | tst.js:14:5:14:8 | f2() | file://:0:0:0:0 | undefined | | tst.js:15:5:15:8 | f2() | file://:0:0:0:0 | undefined | -| tst.js:18:5:18:11 | o1.f3() | file://:0:0:0:0 | indefinite value (call) | +| tst.js:18:5:18:11 | o1.f3() | file://:0:0:0:0 | undefined | | tst.js:23:5:23:8 | f4() | tst.js:21:16:21:30 | function f5 | | tst.js:23:5:23:10 | f4()() | file://:0:0:0:0 | undefined | | tst.js:30:5:30:8 | f6() | tst.js:26:16:28:9 | function f7 | From 676671686785114bf25b72ee925796d4c04ff00b Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 18 Feb 2019 15:00:45 +0100 Subject: [PATCH 16/71] JS: add PropWrite tests for parameter field initializers --- .../ql/test/library-tests/PropWrite/PropWrite.expected | 4 ++++ .../test/library-tests/PropWrite/PropWriteBase.expected | 2 ++ .../library-tests/PropWrite/PropWritePropName.expected | 4 ++++ .../ql/test/library-tests/PropWrite/PropWriteRhs.expected | 4 ++++ javascript/ql/test/library-tests/PropWrite/classes.ts | 8 ++++++++ .../PropWrite/getAPropertyReference.expected | 2 ++ .../PropWrite/getAPropertyReference2.expected | 2 ++ .../library-tests/PropWrite/getAPropertySource.expected | 3 +++ .../library-tests/PropWrite/getAPropertyWrite.expected | 2 ++ .../library-tests/PropWrite/getAPropertyWrite2.expected | 2 ++ .../library-tests/PropWrite/hasPropertyWrite.expected | 2 ++ 11 files changed, 35 insertions(+) diff --git a/javascript/ql/test/library-tests/PropWrite/PropWrite.expected b/javascript/ql/test/library-tests/PropWrite/PropWrite.expected index 091d4f792b65..fe5fd6f52968 100644 --- a/javascript/ql/test/library-tests/PropWrite/PropWrite.expected +++ b/javascript/ql/test/library-tests/PropWrite/PropWrite.expected @@ -2,6 +2,10 @@ | classes.ts:4:3:4:24 | instanc ... foo(); | | classes.ts:8:3:8:39 | constru ... eld) {} | | classes.ts:8:15:8:35 | public ... erField | +| classes.ts:12:5:12:68 | constru ... + 42; } | +| classes.ts:12:17:12:37 | public ... erField | +| classes.ts:16:5:16:46 | constru ... {}) {} | +| classes.ts:16:17:16:37 | public ... erField | | tst.js:3:5:3:8 | x: 4 | | tst.js:4:5:6:5 | func: f ... ;\\n } | | tst.js:7:5:9:5 | f() {\\n ... ;\\n } | diff --git a/javascript/ql/test/library-tests/PropWrite/PropWriteBase.expected b/javascript/ql/test/library-tests/PropWrite/PropWriteBase.expected index 241e2c7d583f..1d8e78556df6 100644 --- a/javascript/ql/test/library-tests/PropWrite/PropWriteBase.expected +++ b/javascript/ql/test/library-tests/PropWrite/PropWriteBase.expected @@ -1,5 +1,7 @@ | classes.ts:4:3:4:24 | instanc ... foo(); | classes.ts:3:21:3:20 | this | | classes.ts:8:15:8:35 | public ... erField | classes.ts:8:3:8:2 | this | +| classes.ts:12:17:12:37 | public ... erField | classes.ts:12:5:12:4 | this | +| classes.ts:16:17:16:37 | public ... erField | classes.ts:16:5:16:4 | this | | tst.js:3:5:3:8 | x: 4 | tst.js:2:11:10:1 | {\\n x ... }\\n} | | tst.js:4:5:6:5 | func: f ... ;\\n } | tst.js:2:11:10:1 | {\\n x ... }\\n} | | tst.js:7:5:9:5 | f() {\\n ... ;\\n } | tst.js:2:11:10:1 | {\\n x ... }\\n} | diff --git a/javascript/ql/test/library-tests/PropWrite/PropWritePropName.expected b/javascript/ql/test/library-tests/PropWrite/PropWritePropName.expected index d2ac2679d25e..f00531e92e6d 100644 --- a/javascript/ql/test/library-tests/PropWrite/PropWritePropName.expected +++ b/javascript/ql/test/library-tests/PropWrite/PropWritePropName.expected @@ -2,6 +2,10 @@ | classes.ts:4:3:4:24 | instanc ... foo(); | instanceField | | classes.ts:8:3:8:39 | constru ... eld) {} | constructor | | classes.ts:8:15:8:35 | public ... erField | parameterField | +| classes.ts:12:5:12:68 | constru ... + 42; } | constructor | +| classes.ts:12:17:12:37 | public ... erField | parameterField | +| classes.ts:16:5:16:46 | constru ... {}) {} | constructor | +| classes.ts:16:17:16:37 | public ... erField | parameterField | | tst.js:3:5:3:8 | x: 4 | x | | tst.js:4:5:6:5 | func: f ... ;\\n } | func | | tst.js:7:5:9:5 | f() {\\n ... ;\\n } | f | diff --git a/javascript/ql/test/library-tests/PropWrite/PropWriteRhs.expected b/javascript/ql/test/library-tests/PropWrite/PropWriteRhs.expected index 68c111317538..48eacf21bc54 100644 --- a/javascript/ql/test/library-tests/PropWrite/PropWriteRhs.expected +++ b/javascript/ql/test/library-tests/PropWrite/PropWriteRhs.expected @@ -2,6 +2,10 @@ | classes.ts:4:3:4:24 | instanc ... foo(); | classes.ts:4:19:4:23 | foo() | | classes.ts:8:3:8:39 | constru ... eld) {} | classes.ts:8:3:8:39 | constru ... eld) {} | | classes.ts:8:15:8:35 | public ... erField | classes.ts:8:22:8:35 | parameterField | +| classes.ts:12:5:12:68 | constru ... + 42; } | classes.ts:12:5:12:68 | constru ... + 42; } | +| classes.ts:12:17:12:37 | public ... erField | classes.ts:12:24:12:37 | parameterField | +| classes.ts:16:5:16:46 | constru ... {}) {} | classes.ts:16:5:16:46 | constru ... {}) {} | +| classes.ts:16:17:16:37 | public ... erField | classes.ts:16:24:16:37 | parameterField | | tst.js:3:5:3:8 | x: 4 | tst.js:3:8:3:8 | 4 | | tst.js:4:5:6:5 | func: f ... ;\\n } | tst.js:4:11:6:5 | functio ... ;\\n } | | tst.js:7:5:9:5 | f() {\\n ... ;\\n } | tst.js:7:6:9:5 | () {\\n ... ;\\n } | diff --git a/javascript/ql/test/library-tests/PropWrite/classes.ts b/javascript/ql/test/library-tests/PropWrite/classes.ts index 7fff3a522c16..fed5f11c43e6 100644 --- a/javascript/ql/test/library-tests/PropWrite/classes.ts +++ b/javascript/ql/test/library-tests/PropWrite/classes.ts @@ -7,3 +7,11 @@ class InstanceField { class ParameterField { constructor(public parameterField) {} } + +class ParameterFieldInit { + constructor(public parameterField = {}) { parameterField + 42; } +} + +class ParameterFieldInitUnused { + constructor(public parameterField = {}) {} +} diff --git a/javascript/ql/test/library-tests/PropWrite/getAPropertyReference.expected b/javascript/ql/test/library-tests/PropWrite/getAPropertyReference.expected index bf832301b9b4..a6692956438c 100644 --- a/javascript/ql/test/library-tests/PropWrite/getAPropertyReference.expected +++ b/javascript/ql/test/library-tests/PropWrite/getAPropertyReference.expected @@ -1,5 +1,7 @@ | classes.ts:3:21:3:20 | this | classes.ts:4:3:4:24 | instanc ... foo(); | | classes.ts:8:3:8:2 | this | classes.ts:8:15:8:35 | public ... erField | +| classes.ts:12:5:12:4 | this | classes.ts:12:17:12:37 | public ... erField | +| classes.ts:16:5:16:4 | this | classes.ts:16:17:16:37 | public ... erField | | tst.js:1:1:1:0 | this | tst.js:23:15:23:29 | this.someMethod | | tst.js:1:1:1:0 | this | tst.js:24:36:24:45 | this.state | | tst.js:2:11:10:1 | {\\n x ... }\\n} | tst.js:3:5:3:8 | x: 4 | diff --git a/javascript/ql/test/library-tests/PropWrite/getAPropertyReference2.expected b/javascript/ql/test/library-tests/PropWrite/getAPropertyReference2.expected index 55c3bb6ea681..8bf36ce9ae77 100644 --- a/javascript/ql/test/library-tests/PropWrite/getAPropertyReference2.expected +++ b/javascript/ql/test/library-tests/PropWrite/getAPropertyReference2.expected @@ -1,5 +1,7 @@ | classes.ts:3:21:3:20 | this | instanceField | classes.ts:4:3:4:24 | instanc ... foo(); | | classes.ts:8:3:8:2 | this | parameterField | classes.ts:8:15:8:35 | public ... erField | +| classes.ts:12:5:12:4 | this | parameterField | classes.ts:12:17:12:37 | public ... erField | +| classes.ts:16:5:16:4 | this | parameterField | classes.ts:16:17:16:37 | public ... erField | | tst.js:1:1:1:0 | this | someMethod | tst.js:23:15:23:29 | this.someMethod | | tst.js:1:1:1:0 | this | state | tst.js:24:36:24:45 | this.state | | tst.js:2:11:10:1 | {\\n x ... }\\n} | f | tst.js:7:5:9:5 | f() {\\n ... ;\\n } | diff --git a/javascript/ql/test/library-tests/PropWrite/getAPropertySource.expected b/javascript/ql/test/library-tests/PropWrite/getAPropertySource.expected index 6cdb0154c11d..54f645ee4ccd 100644 --- a/javascript/ql/test/library-tests/PropWrite/getAPropertySource.expected +++ b/javascript/ql/test/library-tests/PropWrite/getAPropertySource.expected @@ -1,5 +1,8 @@ | classes.ts:3:21:3:20 | this | instanceField | classes.ts:4:19:4:23 | foo() | | classes.ts:8:3:8:2 | this | parameterField | classes.ts:8:22:8:35 | parameterField | +| classes.ts:12:5:12:4 | this | parameterField | classes.ts:12:24:12:37 | parameterField | +| classes.ts:12:5:12:4 | this | parameterField | classes.ts:12:41:12:42 | {} | +| classes.ts:16:5:16:4 | this | parameterField | classes.ts:16:24:16:37 | parameterField | | tst.js:2:11:10:1 | {\\n x ... }\\n} | f | tst.js:7:6:9:5 | () {\\n ... ;\\n } | | tst.js:2:11:10:1 | {\\n x ... }\\n} | func | tst.js:4:11:6:5 | functio ... ;\\n } | | tst.js:12:1:19:1 | class C ... ;\\n }\\n} | func | tst.js:13:14:15:3 | (x) {\\n ... x);\\n } | diff --git a/javascript/ql/test/library-tests/PropWrite/getAPropertyWrite.expected b/javascript/ql/test/library-tests/PropWrite/getAPropertyWrite.expected index 39344ef11d7b..485beb857984 100644 --- a/javascript/ql/test/library-tests/PropWrite/getAPropertyWrite.expected +++ b/javascript/ql/test/library-tests/PropWrite/getAPropertyWrite.expected @@ -1,5 +1,7 @@ | classes.ts:3:21:3:20 | this | classes.ts:4:3:4:24 | instanc ... foo(); | | classes.ts:8:3:8:2 | this | classes.ts:8:15:8:35 | public ... erField | +| classes.ts:12:5:12:4 | this | classes.ts:12:17:12:37 | public ... erField | +| classes.ts:16:5:16:4 | this | classes.ts:16:17:16:37 | public ... erField | | tst.js:2:11:10:1 | {\\n x ... }\\n} | tst.js:3:5:3:8 | x: 4 | | tst.js:2:11:10:1 | {\\n x ... }\\n} | tst.js:4:5:6:5 | func: f ... ;\\n } | | tst.js:2:11:10:1 | {\\n x ... }\\n} | tst.js:7:5:9:5 | f() {\\n ... ;\\n } | diff --git a/javascript/ql/test/library-tests/PropWrite/getAPropertyWrite2.expected b/javascript/ql/test/library-tests/PropWrite/getAPropertyWrite2.expected index 4e38837a447b..e276e476ec83 100644 --- a/javascript/ql/test/library-tests/PropWrite/getAPropertyWrite2.expected +++ b/javascript/ql/test/library-tests/PropWrite/getAPropertyWrite2.expected @@ -1,5 +1,7 @@ | classes.ts:3:21:3:20 | this | instanceField | classes.ts:4:3:4:24 | instanc ... foo(); | | classes.ts:8:3:8:2 | this | parameterField | classes.ts:8:15:8:35 | public ... erField | +| classes.ts:12:5:12:4 | this | parameterField | classes.ts:12:17:12:37 | public ... erField | +| classes.ts:16:5:16:4 | this | parameterField | classes.ts:16:17:16:37 | public ... erField | | tst.js:2:11:10:1 | {\\n x ... }\\n} | f | tst.js:7:5:9:5 | f() {\\n ... ;\\n } | | tst.js:2:11:10:1 | {\\n x ... }\\n} | func | tst.js:4:5:6:5 | func: f ... ;\\n } | | tst.js:2:11:10:1 | {\\n x ... }\\n} | x | tst.js:3:5:3:8 | x: 4 | diff --git a/javascript/ql/test/library-tests/PropWrite/hasPropertyWrite.expected b/javascript/ql/test/library-tests/PropWrite/hasPropertyWrite.expected index b4d999ab9a29..4a8c9979f558 100644 --- a/javascript/ql/test/library-tests/PropWrite/hasPropertyWrite.expected +++ b/javascript/ql/test/library-tests/PropWrite/hasPropertyWrite.expected @@ -1,5 +1,7 @@ | classes.ts:3:21:3:20 | this | instanceField | classes.ts:4:19:4:23 | foo() | | classes.ts:8:3:8:2 | this | parameterField | classes.ts:8:22:8:35 | parameterField | +| classes.ts:12:5:12:4 | this | parameterField | classes.ts:12:24:12:37 | parameterField | +| classes.ts:16:5:16:4 | this | parameterField | classes.ts:16:24:16:37 | parameterField | | tst.js:2:11:10:1 | {\\n x ... }\\n} | f | tst.js:7:6:9:5 | () {\\n ... ;\\n } | | tst.js:2:11:10:1 | {\\n x ... }\\n} | func | tst.js:4:11:6:5 | functio ... ;\\n } | | tst.js:2:11:10:1 | {\\n x ... }\\n} | x | tst.js:3:8:3:8 | 4 | From 6c1b29e4b6f467b78f3e9fa8a800be24a5eac4f8 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 18 Feb 2019 15:05:03 +0100 Subject: [PATCH 17/71] JS: add missing flowstep for unused parameter field initializers --- .../ql/src/semmle/javascript/dataflow/DataFlow.qll | 13 ++++++++++++- .../library-tests/PropWrite/PropWriteRhs.expected | 1 + .../PropWrite/getAPropertySource.expected | 1 + .../PropWrite/hasPropertyWrite.expected | 1 + 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll b/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll index 647f4c7a2378..5c0482ab9df9 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll @@ -558,7 +558,18 @@ module DataFlow { override string getPropertyName() { result = prop.getName() } - override Node getRhs() { result = parameterNode(prop.getParameter()) } + override Node getRhs() { + exists(Parameter param, Node paramNode | + param = prop.getParameter() and + parameterNode(paramNode, param) + | + result = paramNode + or + // special case: there is no SSA flow step for unused parameters + paramNode instanceof UnusedParameterNode and + result = param.getDefault().flow() + ) + } override ControlFlowNode getWriteNode() { result = prop.getParameter() } } diff --git a/javascript/ql/test/library-tests/PropWrite/PropWriteRhs.expected b/javascript/ql/test/library-tests/PropWrite/PropWriteRhs.expected index 48eacf21bc54..42087edc9c1e 100644 --- a/javascript/ql/test/library-tests/PropWrite/PropWriteRhs.expected +++ b/javascript/ql/test/library-tests/PropWrite/PropWriteRhs.expected @@ -6,6 +6,7 @@ | classes.ts:12:17:12:37 | public ... erField | classes.ts:12:24:12:37 | parameterField | | classes.ts:16:5:16:46 | constru ... {}) {} | classes.ts:16:5:16:46 | constru ... {}) {} | | classes.ts:16:17:16:37 | public ... erField | classes.ts:16:24:16:37 | parameterField | +| classes.ts:16:17:16:37 | public ... erField | classes.ts:16:41:16:42 | {} | | tst.js:3:5:3:8 | x: 4 | tst.js:3:8:3:8 | 4 | | tst.js:4:5:6:5 | func: f ... ;\\n } | tst.js:4:11:6:5 | functio ... ;\\n } | | tst.js:7:5:9:5 | f() {\\n ... ;\\n } | tst.js:7:6:9:5 | () {\\n ... ;\\n } | diff --git a/javascript/ql/test/library-tests/PropWrite/getAPropertySource.expected b/javascript/ql/test/library-tests/PropWrite/getAPropertySource.expected index 54f645ee4ccd..101dba0e8a49 100644 --- a/javascript/ql/test/library-tests/PropWrite/getAPropertySource.expected +++ b/javascript/ql/test/library-tests/PropWrite/getAPropertySource.expected @@ -3,6 +3,7 @@ | classes.ts:12:5:12:4 | this | parameterField | classes.ts:12:24:12:37 | parameterField | | classes.ts:12:5:12:4 | this | parameterField | classes.ts:12:41:12:42 | {} | | classes.ts:16:5:16:4 | this | parameterField | classes.ts:16:24:16:37 | parameterField | +| classes.ts:16:5:16:4 | this | parameterField | classes.ts:16:41:16:42 | {} | | tst.js:2:11:10:1 | {\\n x ... }\\n} | f | tst.js:7:6:9:5 | () {\\n ... ;\\n } | | tst.js:2:11:10:1 | {\\n x ... }\\n} | func | tst.js:4:11:6:5 | functio ... ;\\n } | | tst.js:12:1:19:1 | class C ... ;\\n }\\n} | func | tst.js:13:14:15:3 | (x) {\\n ... x);\\n } | diff --git a/javascript/ql/test/library-tests/PropWrite/hasPropertyWrite.expected b/javascript/ql/test/library-tests/PropWrite/hasPropertyWrite.expected index 4a8c9979f558..e6138ec0a33d 100644 --- a/javascript/ql/test/library-tests/PropWrite/hasPropertyWrite.expected +++ b/javascript/ql/test/library-tests/PropWrite/hasPropertyWrite.expected @@ -2,6 +2,7 @@ | classes.ts:8:3:8:2 | this | parameterField | classes.ts:8:22:8:35 | parameterField | | classes.ts:12:5:12:4 | this | parameterField | classes.ts:12:24:12:37 | parameterField | | classes.ts:16:5:16:4 | this | parameterField | classes.ts:16:24:16:37 | parameterField | +| classes.ts:16:5:16:4 | this | parameterField | classes.ts:16:41:16:42 | {} | | tst.js:2:11:10:1 | {\\n x ... }\\n} | f | tst.js:7:6:9:5 | () {\\n ... ;\\n } | | tst.js:2:11:10:1 | {\\n x ... }\\n} | func | tst.js:4:11:6:5 | functio ... ;\\n } | | tst.js:2:11:10:1 | {\\n x ... }\\n} | x | tst.js:3:8:3:8 | 4 | From 9bb7816a3ce375daa955f77da1d24c5e16c01903 Mon Sep 17 00:00:00 2001 From: Raul Garcia Date: Fri, 22 Feb 2019 10:10:20 -0800 Subject: [PATCH 18/71] Making changes based on feedback. --- ...ansform.cs => ThreadUnSafeICryptoTransformBad.cs} | 0 ...ormFix.cs => ThreadUnSafeICryptoTransformGood.cs} | 0 .../Likely Bugs/ThreadUnsafeICryptoTransform.qhelp | 4 ++-- .../src/Likely Bugs/ThreadUnsafeICryptoTransform.ql | 12 ++---------- .../ThreadUnsafeICryptoTransform.cs | 10 +++++----- .../ThreadUnsafeICryptoTransform.expected | 3 +-- 6 files changed, 10 insertions(+), 19 deletions(-) rename csharp/ql/src/Likely Bugs/{ThreadUnsafeICryptoTransform.cs => ThreadUnSafeICryptoTransformBad.cs} (100%) rename csharp/ql/src/Likely Bugs/{ThreadUnSafeICryptoTransformFix.cs => ThreadUnSafeICryptoTransformGood.cs} (100%) diff --git a/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.cs b/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformBad.cs similarity index 100% rename from csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.cs rename to csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformBad.cs diff --git a/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformFix.cs b/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformGood.cs similarity index 100% rename from csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformFix.cs rename to csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformGood.cs diff --git a/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp index 0f1ae02868d0..1e71a53840ee 100644 --- a/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp +++ b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp @@ -16,10 +16,10 @@

    This example demonstrates the dangers of using a static System.Security.Cryptography.ICryptoTransform in such a way that the results may be incorrect.

    - +

    A simple fix is to change the _sha field from being a static member to an instance one by removing the static keyword.

    - +     diff --git a/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.ql b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.ql index 6db9fad3b45b..e63cfa66c1e1 100644 --- a/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.ql +++ b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.ql @@ -23,13 +23,13 @@ class ICryptoTransform extends Class { predicate usesICryptoTransformType( Type t ) { exists( ICryptoTransform ict | ict = t - or usesICryptoTransformType( t.getAChild*() ) + or usesICryptoTransformType( t.getAChild() ) ) } predicate hasICryptoTransformMember( Class c) { exists( Field f | - f = c.getAMember*() + f = c.getAMember() and ( exists( ICryptoTransform ict | ict = f.getType() ) or hasICryptoTransformMember(f.getType()) @@ -75,14 +75,6 @@ predicate hasICryptoTransformStaticMember( Class c, string msg) { ) ) ) - or - exists( Field f | - f = c.getAMember*() - and not f.isStatic() - and ( hasICryptoTransformStaticMember( f.getType(), _ ) - and msg = "Non-static field " + f + " of type " + f.getType() + " internally makes use of an static object that implements 'System.Security.Cryptography.ICryptoTransform'. This causes that usage of this class member is unsafe for concurrent threads." - ) - ) or ( hasICryptoTransformStaticMemberNested(c) and msg = "Class" + c + " implementation depends on a static object of type 'System.Security.Cryptography.ICryptoTransform' in a way that is unsafe for concurrent threads." ) diff --git a/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.cs b/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.cs index 90214fc988a2..0983c9c768db 100644 --- a/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.cs +++ b/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.cs @@ -66,11 +66,6 @@ public class StaticMember private static SHA1 _sha1 = SHA1.Create(); } -public class IndirectStatic -{ - StaticMember tc; -} - public class IndirectStatic2 { static Nest02 _n = new Nest02(); @@ -80,6 +75,11 @@ public class IndirectStatic2 /// Should not be flagged (thread safe) /// +public class IndirectStatic +{ + StaticMember tc; +} + public class TokenCacheFP { /// diff --git a/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.expected b/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.expected index e9ca1952f635..defba765fa98 100644 --- a/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.expected +++ b/csharp/ql/test/query-tests/Likely Bugs/ThreadUnsafeICryptoTransform/ThreadUnsafeICryptoTransform.expected @@ -2,5 +2,4 @@ | ThreadUnsafeICryptoTransform.cs:44:14:44:19 | Nest04 | ClassNest04 implementation depends on a static object of type 'System.Security.Cryptography.ICryptoTransform' in a way that is unsafe for concurrent threads. | | ThreadUnsafeICryptoTransform.cs:49:21:49:42 | StaticMemberChildUsage | Static field HashMap of type Dictionary makes usage of 'System.Security.Cryptography.ICryptoTransform', but it does not have an attribute [ThreadStatic]. The usage of this class is unsafe for concurrent threads. | | ThreadUnsafeICryptoTransform.cs:64:14:64:25 | StaticMember | Static field _sha1 of type SHA1, implements 'System.Security.Cryptography.ICryptoTransform', but it does not have an attribute [ThreadStatic]. The usage of this class is unsafe for concurrent threads. | -| ThreadUnsafeICryptoTransform.cs:69:14:69:27 | IndirectStatic | Non-static field tc of type StaticMember internally makes use of an static object that implements 'System.Security.Cryptography.ICryptoTransform'. This causes that usage of this class member is unsafe for concurrent threads. | -| ThreadUnsafeICryptoTransform.cs:74:14:74:28 | IndirectStatic2 | ClassIndirectStatic2 implementation depends on a static object of type 'System.Security.Cryptography.ICryptoTransform' in a way that is unsafe for concurrent threads. | +| ThreadUnsafeICryptoTransform.cs:69:14:69:28 | IndirectStatic2 | ClassIndirectStatic2 implementation depends on a static object of type 'System.Security.Cryptography.ICryptoTransform' in a way that is unsafe for concurrent threads. | From 047b69a4c2fd73de2fc737802ad142cd40121e66 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 25 Feb 2019 15:08:56 +0100 Subject: [PATCH 19/71] JS: address review comments --- .../ql/src/Declarations/UnusedParameter.qll | 2 +- .../ql/src/Declarations/UnusedProperty.ql | 30 +++++++++++-------- .../ql/src/Declarations/UnusedVariable.qll | 2 +- .../ql/src/Expressions/ExprHasNoEffect.qll | 2 +- .../javascript/dataflow/CapturedNodes.qll | 4 +-- .../internal/InterProceduralTypeInference.qll | 18 ++++++----- 6 files changed, 32 insertions(+), 26 deletions(-) diff --git a/javascript/ql/src/Declarations/UnusedParameter.qll b/javascript/ql/src/Declarations/UnusedParameter.qll index b4fb5cb578a7..b8ff6b1d38e5 100644 --- a/javascript/ql/src/Declarations/UnusedParameter.qll +++ b/javascript/ql/src/Declarations/UnusedParameter.qll @@ -1,5 +1,5 @@ /** - * This library contains the majority of the 'js/unused-parameter' query implementation. + * Provides classes and predicates for the 'js/unused-parameter' query. * * In order to suppress alerts that are similar to the 'js/unused-parameter' alerts, * `isAnAccidentallyUnusedParameter` should be used since it holds iff that alert is active. diff --git a/javascript/ql/src/Declarations/UnusedProperty.ql b/javascript/ql/src/Declarations/UnusedProperty.ql index b364a2b21483..117d402e0dff 100644 --- a/javascript/ql/src/Declarations/UnusedProperty.ql +++ b/javascript/ql/src/Declarations/UnusedProperty.ql @@ -30,9 +30,12 @@ predicate hasUnknownPropertyRead(CapturedSource obj) { exists(obj.getAPropertyRead("propertyIsEnumerable")) } -predicate flowsToTypeRestrictedExpression(CapturedSource n) { +/** + * Holds if `obj` flows to an expression that must have a specific type. + */ +predicate flowsToTypeRestrictedExpression(CapturedSource obj) { exists (Expr restricted, TypeExpr type | - n.flowsToExpr(restricted) and + obj.flowsToExpr(restricted) and not type.isAny() | exists (TypeAssertion assertion | type = assertion.getTypeAnnotation() and @@ -47,14 +50,15 @@ predicate flowsToTypeRestrictedExpression(CapturedSource n) { ) } -from DataFlow::PropWrite w, CapturedSource n, string name +from DataFlow::PropWrite write, CapturedSource obj, string name where - w = n.getAPropertyWrite(name) and - not exists(n.getAPropertyRead(name)) and - not w.getBase().analyze().getAValue() != n.analyze().getAValue() and - not hasUnknownPropertyRead(n) and + write = obj.getAPropertyWrite(name) and + not exists(obj.getAPropertyRead(name)) and + // `obj` is the only base object for the write: it is not spurious + not write.getBase().analyze().getAValue() != obj.analyze().getAValue() and + not hasUnknownPropertyRead(obj) and // avoid reporting if the definition is unreachable - w.getAstNode().getFirstControlFlowNode().getBasicBlock() instanceof ReachableBasicBlock and + write.getAstNode().getFirstControlFlowNode().getBasicBlock() instanceof ReachableBasicBlock and // avoid implicitly read properties not ( name = "toString" or @@ -62,13 +66,13 @@ where name.matches("@@%") // @@iterator, for example ) and // avoid flagging properties that a type system requires - not flowsToTypeRestrictedExpression(n) and + not flowsToTypeRestrictedExpression(obj) and // flagged by js/unused-local-variable - not exists(UnusedLocal l | l.getAnAssignedExpr().getUnderlyingValue().flow() = n) and + not exists(UnusedLocal l | l.getAnAssignedExpr().getUnderlyingValue().flow() = obj) and // flagged by js/unused-parameter not exists(Parameter p | isAnAccidentallyUnusedParameter(p) | - p.getDefault().getUnderlyingValue().flow() = n + p.getDefault().getUnderlyingValue().flow() = obj ) and // flagged by js/useless-expression - not inVoidContext(n.asExpr()) -select w, "Unused property " + name + "." + not inVoidContext(obj.asExpr()) +select write, "Unused property " + name + "." diff --git a/javascript/ql/src/Declarations/UnusedVariable.qll b/javascript/ql/src/Declarations/UnusedVariable.qll index 98d69b80938c..143a8c5e0746 100644 --- a/javascript/ql/src/Declarations/UnusedVariable.qll +++ b/javascript/ql/src/Declarations/UnusedVariable.qll @@ -1,5 +1,5 @@ /** - * This library contains parts of the 'js/unused-local-variable' query implementation. + * Provides classes and predicates for the 'js/unused-local-variable' query. */ import javascript diff --git a/javascript/ql/src/Expressions/ExprHasNoEffect.qll b/javascript/ql/src/Expressions/ExprHasNoEffect.qll index 3b6f4703ad6e..858f719ba0a0 100644 --- a/javascript/ql/src/Expressions/ExprHasNoEffect.qll +++ b/javascript/ql/src/Expressions/ExprHasNoEffect.qll @@ -1,5 +1,5 @@ /** - * This library contains parts of the 'js/useless-expression' query implementation. + * Provides classes and predicates for the 'js/useless-expression' query. */ import javascript diff --git a/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll b/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll index bcb9c44abe5a..db4a0c309399 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll @@ -42,7 +42,7 @@ private predicate exposedAsReceiver(DataFlow::SourceNode n) { n.flowsToExpr(any(FunctionBindExpr bind).getObject()) or // technically, the builtin prototypes could have a `this`-using function through which this node escapes, but we ignore that here - // (we also ignore `o['__' + 'proto__"] = ...`) + // (we also ignore `o['__' + 'proto__'] = ...`) exists(n.getAPropertyWrite("__proto__")) or // could check the assigned value of all affected variables, but it is unlikely to matter in practice @@ -51,7 +51,7 @@ private predicate exposedAsReceiver(DataFlow::SourceNode n) { /** * A source for which the flow is entirely captured by the dataflow library. - * All uses of the node is represented by `this.flowsTo(_)` and friends. + * All uses of the node are modeled by `this.flowsTo(_)` and related predicates. */ class CapturedSource extends DataFlow::SourceNode { CapturedSource() { diff --git a/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll b/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll index fcb3e00fb255..9574f7caad2e 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll @@ -256,20 +256,22 @@ private predicate hasDefiniteReceiver( * Enables inter-procedural type inference for the return value of a * method call to a flow-insensitively type-inferred callee. */ -class TypeInferredMethodWithAnalyzedReturnFlow extends CallWithNonLocalAnalyzedReturnFlow { +private class TypeInferredMethodWithAnalyzedReturnFlow extends CallWithNonLocalAnalyzedReturnFlow { DataFlow::FunctionNode fun; TypeInferredMethodWithAnalyzedReturnFlow() { - exists(CapturedSource s, DataFlow::PropWrite w, string name | + exists(CapturedSource obj, DataFlow::PropWrite write, string name | this.(DataFlow::MethodCallNode).getMethodName() = name and - s.hasOwnProperty(name) and - hasDefiniteReceiver(this, s) and - w = s.getAPropertyWrite() and - fun.flowsTo(w.getRhs()) and + obj.hasOwnProperty(name) and + hasDefiniteReceiver(this, obj) and + // include all potential callees + // by construction, there are no unknown methods on `obj` + write = obj.getAPropertyWrite() and + fun.flowsTo(write.getRhs()) and ( - not exists(w.getPropertyName()) + not exists(write.getPropertyName()) or - w.getPropertyName() = name + write.getPropertyName() = name ) ) } From 0d94fe3f540e9e18bdcc5659a14bf709f8e46718 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 25 Feb 2019 15:31:53 +0100 Subject: [PATCH 20/71] JS: analyze assignments in `with` correctly --- .../ql/src/semmle/javascript/dataflow/CapturedNodes.qll | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll b/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll index db4a0c309399..85cbd3ae5fb0 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll @@ -20,7 +20,11 @@ private predicate isEscape(DataFlow::Node escape, string cause) { or escape = any(ExportDeclaration e).getSourceNode(_) and cause = "export" or - any(WithStmt with).mayAffect(escape.asExpr()) and cause = "heap" + exists (WithStmt with, Assignment assign | + with.mayAffect(assign.getLhs()) and + assign.getRhs().flow() = escape and + cause = "heap" + ) } private DataFlow::Node getAnEscape() { From 1150f4c02b17e1bb4124142a60b33a85782ec11e Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 25 Feb 2019 15:52:23 +0100 Subject: [PATCH 21/71] JS: add documentation to test case --- .../CapturedNodes/CapturedSource.expected | 2 +- .../CapturedSource_hasOwnProperty.expected | 2 +- .../CapturedNodes/method-calls.js | 28 +++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected index d78dd6e228ff..c0e0331413f5 100644 --- a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected +++ b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected @@ -4,7 +4,7 @@ | method-calls.js:23:11:23:17 | {m: f1} | | method-calls.js:27:11:27:17 | {m: f2} | | method-calls.js:32:12:32:18 | {m: f3} | -| method-calls.js:36:15:36:21 | {m: f2} | +| method-calls.js:36:15:36:21 | {m: f4} | | method-calls.js:42:16:42:28 | {m: () => 42} | | method-calls.js:46:17:46:29 | {m: () => 42} | | method-calls.js:50:16:50:28 | {m: () => 42} | diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected index 76e494cf1ad0..3fb32b5407ed 100644 --- a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected +++ b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected @@ -5,7 +5,7 @@ | method-calls.js:23:11:23:17 | {m: f1} | m | | method-calls.js:27:11:27:17 | {m: f2} | m | | method-calls.js:32:12:32:18 | {m: f3} | m | -| method-calls.js:36:15:36:21 | {m: f2} | m | +| method-calls.js:36:15:36:21 | {m: f4} | m | | method-calls.js:42:16:42:28 | {m: () => 42} | m | | method-calls.js:46:17:46:29 | {m: () => 42} | m | | method-calls.js:50:16:50:28 | {m: () => 42} | m | diff --git a/javascript/ql/test/library-tests/CapturedNodes/method-calls.js b/javascript/ql/test/library-tests/CapturedNodes/method-calls.js index f17f72e58962..bee0b6a58030 100644 --- a/javascript/ql/test/library-tests/CapturedNodes/method-calls.js +++ b/javascript/ql/test/library-tests/CapturedNodes/method-calls.js @@ -5,53 +5,53 @@ m3: function(){ return {}; }, m4: function(){ return {}; } }; - o1.m1(); - o1.m2(); + o1.m1(); // analyzed precisely + o1.m2(); // analyzed precisely unknown(o1.m2); var o = unknown? o1: {}; - o.m3(); // NOT supported + o.m3(); // not analyzed precisely: `o1` is not the only receiver. var m4 = o.m4; - m4(); + m4(); // (not a method call) var o2 = {}; o2.m = function() { return {}; }; - o2[unknown] = function() { return true; }; // could be __proto__ + // not analyzed precisely: `m` may be in the prototype since `m` is not in the initializer, and we do not attempt to reason flow sensitively beyond that at the moment o2.m(); }); (function(){ function f1(){return {};} - var v1 = {m: f1}.m(); + var v1 = {m: f1}.m(); // analyzed precisely v1 === true; function f2(){return {};} var o2 = {m: f2}; - var v2 = o2.m(); + var v2 = o2.m(); // analyzed precisely v2 === true; function f3(){return {};} - var v3 = ({m: f3}).m(); + var v3 = ({m: f3}).m(); // analyzed precisely v3 === true; function f4(){return {};} - var { o4 } = {m: f2}; - var v4 = o4.m(); + var { o4 } = {m: f4}; + var v4 = o4.m(); // not analyzed precisely: o4 is from a destructuring assignment (and is even `undefined` in this case v4 === true; }); (function(){ (function(o = {m: () => 42}){ - var v1 = o.m(); + var v1 = o.m(); // not analyzed precisely: `o` may be `unknown` })(unknown); function f(o = {m: () => 42}){ - var v2 = o.m(); + var v2 = o.m(); // not analyzed precisely: `o` may be `unknown` }; f(unknown); (function(o = {m: () => 42}){ - var v3 = o.m(); + var v3 = o.m(); // not analyzed precisely: `o.m` may be `unknown` })({m: unknown}); (function(o = {m: () => 42}){ - var v4 = o.m(); + var v4 = o.m(); // not analyzed precisely: we only support unique receivers at the moment })({m: () => true}); }); From 65fb1423b72994279391e2883241c2259504de80 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 25 Feb 2019 15:55:44 +0100 Subject: [PATCH 22/71] JS: format test case (update expected output) --- .../CapturedNodes/CapturedSource.expected | 16 ++++++++-------- .../CapturedSource_hasOwnProperty.expected | 16 ++++++++-------- .../MethodCallTypeInference.expected | 14 +++++++------- .../MethodCallTypeInferenceUsage.expected | 8 ++++---- .../library-tests/CapturedNodes/method-calls.js | 12 +++++++++--- 5 files changed, 36 insertions(+), 30 deletions(-) diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected index c0e0331413f5..173282ed749b 100644 --- a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected +++ b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected @@ -1,14 +1,14 @@ | method-calls.js:2:11:7:2 | {\\n\\t\\tm1: ... }; }\\n\\t} | | method-calls.js:11:23:11:24 | {} | | method-calls.js:16:11:16:12 | {} | -| method-calls.js:23:11:23:17 | {m: f1} | -| method-calls.js:27:11:27:17 | {m: f2} | -| method-calls.js:32:12:32:18 | {m: f3} | -| method-calls.js:36:15:36:21 | {m: f4} | -| method-calls.js:42:16:42:28 | {m: () => 42} | -| method-calls.js:46:17:46:29 | {m: () => 42} | -| method-calls.js:50:16:50:28 | {m: () => 42} | -| method-calls.js:53:16:53:28 | {m: () => 42} | +| method-calls.js:25:11:25:17 | {m: f1} | +| method-calls.js:29:11:29:17 | {m: f2} | +| method-calls.js:34:12:34:18 | {m: f3} | +| method-calls.js:38:15:38:21 | {m: f4} | +| method-calls.js:46:16:46:28 | {m: () => 42} | +| method-calls.js:50:17:50:29 | {m: () => 42} | +| method-calls.js:54:16:54:28 | {m: () => 42} | +| method-calls.js:57:16:57:28 | {m: () => 42} | | tst.js:3:18:3:19 | {} | | tst.js:4:18:4:36 | { f: function(){} } | | tst.js:5:18:5:34 | { [unknown]: 42 } | diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected index 3fb32b5407ed..ef9d64c3c70a 100644 --- a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected +++ b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected @@ -2,14 +2,14 @@ | method-calls.js:2:11:7:2 | {\\n\\t\\tm1: ... }; }\\n\\t} | m2 | | method-calls.js:2:11:7:2 | {\\n\\t\\tm1: ... }; }\\n\\t} | m3 | | method-calls.js:2:11:7:2 | {\\n\\t\\tm1: ... }; }\\n\\t} | m4 | -| method-calls.js:23:11:23:17 | {m: f1} | m | -| method-calls.js:27:11:27:17 | {m: f2} | m | -| method-calls.js:32:12:32:18 | {m: f3} | m | -| method-calls.js:36:15:36:21 | {m: f4} | m | -| method-calls.js:42:16:42:28 | {m: () => 42} | m | -| method-calls.js:46:17:46:29 | {m: () => 42} | m | -| method-calls.js:50:16:50:28 | {m: () => 42} | m | -| method-calls.js:53:16:53:28 | {m: () => 42} | m | +| method-calls.js:25:11:25:17 | {m: f1} | m | +| method-calls.js:29:11:29:17 | {m: f2} | m | +| method-calls.js:34:12:34:18 | {m: f3} | m | +| method-calls.js:38:15:38:21 | {m: f4} | m | +| method-calls.js:46:16:46:28 | {m: () => 42} | m | +| method-calls.js:50:17:50:29 | {m: () => 42} | m | +| method-calls.js:54:16:54:28 | {m: () => 42} | m | +| method-calls.js:57:16:57:28 | {m: () => 42} | m | | tst.js:4:18:4:36 | { f: function(){} } | f | | tst.js:38:18:38:26 | { p: 42 } | p | | tst.js:42:18:42:33 | { p: 42, q: 42 } | p | diff --git a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected index 1192bcdb1275..afab24f56bca 100644 --- a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected +++ b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected @@ -1,12 +1,12 @@ | method-calls.js:8:2:8:8 | o1.m1() | object | | method-calls.js:9:2:9:8 | o1.m2() | object | | method-calls.js:12:2:12:7 | o.m3() | boolean, class, date, function, null, number, object, regular expression,string or undefined | -| method-calls.js:19:2:19:7 | o2.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | -| method-calls.js:23:11:23:21 | {m: f1}.m() | object | -| method-calls.js:28:11:28:16 | o2.m() | object | -| method-calls.js:32:11:32:23 | ({m: f3}).m() | object | -| method-calls.js:37:11:37:16 | o4.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | -| method-calls.js:43:12:43:16 | o.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:21:2:21:7 | o2.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:25:11:25:21 | {m: f1}.m() | object | +| method-calls.js:30:11:30:16 | o2.m() | object | +| method-calls.js:34:11:34:23 | ({m: f3}).m() | object | +| method-calls.js:41:11:41:16 | o4.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | | method-calls.js:47:12:47:16 | o.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | | method-calls.js:51:12:51:16 | o.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | -| method-calls.js:54:12:54:16 | o.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:55:12:55:16 | o.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| method-calls.js:60:12:60:16 | o.m() | boolean, class, date, function, null, number, object, regular expression,string or undefined | diff --git a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected index 7c393230c8bc..257fd863f811 100644 --- a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected +++ b/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected @@ -1,4 +1,4 @@ -| method-calls.js:23:11:23:21 | {m: f1}.m() | method-calls.js:24:2:24:3 | v1 | method-calls.js:22:23:22:24 | object literal | -| method-calls.js:28:11:28:16 | o2.m() | method-calls.js:29:2:29:3 | v2 | method-calls.js:26:23:26:24 | object literal | -| method-calls.js:32:11:32:23 | ({m: f3}).m() | method-calls.js:33:2:33:3 | v3 | method-calls.js:31:23:31:24 | object literal | -| method-calls.js:37:11:37:16 | o4.m() | method-calls.js:38:2:38:3 | v4 | file://:0:0:0:0 | indefinite value (call) | +| method-calls.js:25:11:25:21 | {m: f1}.m() | method-calls.js:26:2:26:3 | v1 | method-calls.js:24:23:24:24 | object literal | +| method-calls.js:30:11:30:16 | o2.m() | method-calls.js:31:2:31:3 | v2 | method-calls.js:28:23:28:24 | object literal | +| method-calls.js:34:11:34:23 | ({m: f3}).m() | method-calls.js:35:2:35:3 | v3 | method-calls.js:33:23:33:24 | object literal | +| method-calls.js:41:11:41:16 | o4.m() | method-calls.js:42:2:42:3 | v4 | file://:0:0:0:0 | indefinite value (call) | diff --git a/javascript/ql/test/library-tests/CapturedNodes/method-calls.js b/javascript/ql/test/library-tests/CapturedNodes/method-calls.js index bee0b6a58030..79947632eea3 100644 --- a/javascript/ql/test/library-tests/CapturedNodes/method-calls.js +++ b/javascript/ql/test/library-tests/CapturedNodes/method-calls.js @@ -15,7 +15,9 @@ var o2 = {}; o2.m = function() { return {}; }; - // not analyzed precisely: `m` may be in the prototype since `m` is not in the initializer, and we do not attempt to reason flow sensitively beyond that at the moment + // not analyzed precisely: `m` may be in the prototype since `m` + // is not in the initializer, and we do not attempt to reason flow + // sensitively beyond that at the moment o2.m(); }); (function(){ @@ -34,7 +36,9 @@ function f4(){return {};} var { o4 } = {m: f4}; - var v4 = o4.m(); // not analyzed precisely: o4 is from a destructuring assignment (and is even `undefined` in this case + // not analyzed precisely: o4 is from a destructuring assignment + // (and it is even `undefined` in this case) + var v4 = o4.m(); v4 === true; }); @@ -51,7 +55,9 @@ var v3 = o.m(); // not analyzed precisely: `o.m` may be `unknown` })({m: unknown}); (function(o = {m: () => 42}){ - var v4 = o.m(); // not analyzed precisely: we only support unique receivers at the moment + // not analyzed precisely: we only support unique receivers at + // the moment + var v4 = o.m(); })({m: () => true}); }); From 66367987af24cd5c7fcbfd49383f70bfc1a3c000 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 25 Feb 2019 16:04:37 +0100 Subject: [PATCH 23/71] JS: rename CapturedSource -> LocalObject --- javascript/ql/src/Declarations/UnusedProperty.ql | 6 +++--- .../ql/src/semmle/javascript/dataflow/CapturedNodes.qll | 6 +++--- .../dataflow/internal/InterProceduralTypeInference.qll | 4 ++-- .../ql/test/library-tests/CapturedNodes/CapturedSource.ql | 2 +- .../CapturedNodes/CapturedSource_hasOwnProperty.ql | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/javascript/ql/src/Declarations/UnusedProperty.ql b/javascript/ql/src/Declarations/UnusedProperty.ql index 117d402e0dff..d7208c80c942 100644 --- a/javascript/ql/src/Declarations/UnusedProperty.ql +++ b/javascript/ql/src/Declarations/UnusedProperty.ql @@ -14,7 +14,7 @@ import UnusedVariable import UnusedParameter import Expressions.ExprHasNoEffect -predicate hasUnknownPropertyRead(CapturedSource obj) { +predicate hasUnknownPropertyRead(LocalObject obj) { // dynamic reads exists(DataFlow::PropRead r | obj.getAPropertyRead() = r | not exists(r.getPropertyName())) or @@ -33,7 +33,7 @@ predicate hasUnknownPropertyRead(CapturedSource obj) { /** * Holds if `obj` flows to an expression that must have a specific type. */ -predicate flowsToTypeRestrictedExpression(CapturedSource obj) { +predicate flowsToTypeRestrictedExpression(LocalObject obj) { exists (Expr restricted, TypeExpr type | obj.flowsToExpr(restricted) and not type.isAny() | @@ -50,7 +50,7 @@ predicate flowsToTypeRestrictedExpression(CapturedSource obj) { ) } -from DataFlow::PropWrite write, CapturedSource obj, string name +from DataFlow::PropWrite write, LocalObject obj, string name where write = obj.getAPropertyWrite(name) and not exists(obj.getAPropertyRead(name)) and diff --git a/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll b/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll index 85cbd3ae5fb0..cf7575fcbd45 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll @@ -54,11 +54,11 @@ private predicate exposedAsReceiver(DataFlow::SourceNode n) { } /** - * A source for which the flow is entirely captured by the dataflow library. + * A source that is entirely local, i.e. the dataflow library models all of its flow. * All uses of the node are modeled by `this.flowsTo(_)` and related predicates. */ -class CapturedSource extends DataFlow::SourceNode { - CapturedSource() { +class LocalObject extends DataFlow::SourceNode { + LocalObject() { // pragmatic limitation: object literals only this instanceof DataFlow::ObjectLiteralNode and not flowsTo(getAnEscape()) and diff --git a/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll b/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll index 9574f7caad2e..73d43c4a1a64 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll @@ -238,7 +238,7 @@ private class TypeInferredCalleeWithAnalyzedReturnFlow extends CallWithNonLocalA */ pragma[noinline] private predicate hasDefiniteReceiver( - DataFlow::MethodCallNode call, CapturedSource receiver + DataFlow::MethodCallNode call, LocalObject receiver ) { call = receiver.getAMethodCall() and exists (DataFlow::AnalyzedNode receiverNode, AbstractValue abstractCapturedReceiver | @@ -260,7 +260,7 @@ private class TypeInferredMethodWithAnalyzedReturnFlow extends CallWithNonLocalA DataFlow::FunctionNode fun; TypeInferredMethodWithAnalyzedReturnFlow() { - exists(CapturedSource obj, DataFlow::PropWrite write, string name | + exists(LocalObject obj, DataFlow::PropWrite write, string name | this.(DataFlow::MethodCallNode).getMethodName() = name and obj.hasOwnProperty(name) and hasDefiniteReceiver(this, obj) and diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.ql b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.ql index 0731e04afee1..0a9997259781 100644 --- a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.ql +++ b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.ql @@ -1,4 +1,4 @@ import javascript import semmle.javascript.dataflow.CapturedNodes -select any(CapturedSource n) +select any(LocalObject n) diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.ql b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.ql index 14911b329daa..7c9fe8d469a3 100644 --- a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.ql +++ b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.ql @@ -1,6 +1,6 @@ import javascript import semmle.javascript.dataflow.CapturedNodes -from CapturedSource src, string name +from LocalObject src, string name where src.hasOwnProperty(name) select src, name From 4dc147d506e2ed3da0e2576c405baf0b96846e3c Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 25 Feb 2019 16:09:07 +0100 Subject: [PATCH 24/71] JS: rename CapturedSource -> LocalObject (files) --- javascript/ql/src/Declarations/UnusedProperty.ql | 2 +- .../dataflow/{CapturedNodes.qll => LocalObjects.qll} | 0 .../dataflow/internal/InterProceduralTypeInference.qll | 3 +-- .../ql/test/library-tests/CapturedNodes/CapturedSource.ql | 4 ---- .../LocalObject.expected} | 0 javascript/ql/test/library-tests/LocalObjects/LocalObject.ql | 4 ++++ .../LocalObject_hasOwnProperty.expected} | 0 .../LocalObject_hasOwnProperty.ql} | 2 +- .../MethodCallTypeInference.expected | 0 .../MethodCallTypeInference.ql | 0 .../MethodCallTypeInferenceUsage.expected | 0 .../MethodCallTypeInferenceUsage.ql | 0 .../{CapturedNodes => LocalObjects}/method-calls.js | 0 .../test/library-tests/{CapturedNodes => LocalObjects}/tst.js | 0 .../test/library-tests/{CapturedNodes => LocalObjects}/tst.ts | 0 15 files changed, 7 insertions(+), 8 deletions(-) rename javascript/ql/src/semmle/javascript/dataflow/{CapturedNodes.qll => LocalObjects.qll} (100%) delete mode 100644 javascript/ql/test/library-tests/CapturedNodes/CapturedSource.ql rename javascript/ql/test/library-tests/{CapturedNodes/CapturedSource.expected => LocalObjects/LocalObject.expected} (100%) create mode 100644 javascript/ql/test/library-tests/LocalObjects/LocalObject.ql rename javascript/ql/test/library-tests/{CapturedNodes/CapturedSource_hasOwnProperty.expected => LocalObjects/LocalObject_hasOwnProperty.expected} (100%) rename javascript/ql/test/library-tests/{CapturedNodes/CapturedSource_hasOwnProperty.ql => LocalObjects/LocalObject_hasOwnProperty.ql} (67%) rename javascript/ql/test/library-tests/{CapturedNodes => LocalObjects}/MethodCallTypeInference.expected (100%) rename javascript/ql/test/library-tests/{CapturedNodes => LocalObjects}/MethodCallTypeInference.ql (100%) rename javascript/ql/test/library-tests/{CapturedNodes => LocalObjects}/MethodCallTypeInferenceUsage.expected (100%) rename javascript/ql/test/library-tests/{CapturedNodes => LocalObjects}/MethodCallTypeInferenceUsage.ql (100%) rename javascript/ql/test/library-tests/{CapturedNodes => LocalObjects}/method-calls.js (100%) rename javascript/ql/test/library-tests/{CapturedNodes => LocalObjects}/tst.js (100%) rename javascript/ql/test/library-tests/{CapturedNodes => LocalObjects}/tst.ts (100%) diff --git a/javascript/ql/src/Declarations/UnusedProperty.ql b/javascript/ql/src/Declarations/UnusedProperty.ql index d7208c80c942..8edc34936666 100644 --- a/javascript/ql/src/Declarations/UnusedProperty.ql +++ b/javascript/ql/src/Declarations/UnusedProperty.ql @@ -9,7 +9,7 @@ */ import javascript -import semmle.javascript.dataflow.CapturedNodes +import semmle.javascript.dataflow.LocalObjects import UnusedVariable import UnusedParameter import Expressions.ExprHasNoEffect diff --git a/javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll b/javascript/ql/src/semmle/javascript/dataflow/LocalObjects.qll similarity index 100% rename from javascript/ql/src/semmle/javascript/dataflow/CapturedNodes.qll rename to javascript/ql/src/semmle/javascript/dataflow/LocalObjects.qll diff --git a/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll b/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll index 73d43c4a1a64..436b4966b57c 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll @@ -6,8 +6,7 @@ import javascript import AbstractValuesImpl - -import semmle.javascript.dataflow.CapturedNodes +import semmle.javascript.dataflow.LocalObjects /** * Flow analysis for `this` expressions inside functions. diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.ql b/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.ql deleted file mode 100644 index 0a9997259781..000000000000 --- a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.ql +++ /dev/null @@ -1,4 +0,0 @@ -import javascript -import semmle.javascript.dataflow.CapturedNodes - -select any(LocalObject n) diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected b/javascript/ql/test/library-tests/LocalObjects/LocalObject.expected similarity index 100% rename from javascript/ql/test/library-tests/CapturedNodes/CapturedSource.expected rename to javascript/ql/test/library-tests/LocalObjects/LocalObject.expected diff --git a/javascript/ql/test/library-tests/LocalObjects/LocalObject.ql b/javascript/ql/test/library-tests/LocalObjects/LocalObject.ql new file mode 100644 index 000000000000..d2bfd93b5368 --- /dev/null +++ b/javascript/ql/test/library-tests/LocalObjects/LocalObject.ql @@ -0,0 +1,4 @@ +import javascript +import semmle.javascript.dataflow.LocalObjects + +select any(LocalObject n) diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected b/javascript/ql/test/library-tests/LocalObjects/LocalObject_hasOwnProperty.expected similarity index 100% rename from javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.expected rename to javascript/ql/test/library-tests/LocalObjects/LocalObject_hasOwnProperty.expected diff --git a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.ql b/javascript/ql/test/library-tests/LocalObjects/LocalObject_hasOwnProperty.ql similarity index 67% rename from javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.ql rename to javascript/ql/test/library-tests/LocalObjects/LocalObject_hasOwnProperty.ql index 7c9fe8d469a3..28dfabddb38e 100644 --- a/javascript/ql/test/library-tests/CapturedNodes/CapturedSource_hasOwnProperty.ql +++ b/javascript/ql/test/library-tests/LocalObjects/LocalObject_hasOwnProperty.ql @@ -1,5 +1,5 @@ import javascript -import semmle.javascript.dataflow.CapturedNodes +import semmle.javascript.dataflow.LocalObjects from LocalObject src, string name where src.hasOwnProperty(name) diff --git a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected b/javascript/ql/test/library-tests/LocalObjects/MethodCallTypeInference.expected similarity index 100% rename from javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.expected rename to javascript/ql/test/library-tests/LocalObjects/MethodCallTypeInference.expected diff --git a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.ql b/javascript/ql/test/library-tests/LocalObjects/MethodCallTypeInference.ql similarity index 100% rename from javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInference.ql rename to javascript/ql/test/library-tests/LocalObjects/MethodCallTypeInference.ql diff --git a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected b/javascript/ql/test/library-tests/LocalObjects/MethodCallTypeInferenceUsage.expected similarity index 100% rename from javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.expected rename to javascript/ql/test/library-tests/LocalObjects/MethodCallTypeInferenceUsage.expected diff --git a/javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.ql b/javascript/ql/test/library-tests/LocalObjects/MethodCallTypeInferenceUsage.ql similarity index 100% rename from javascript/ql/test/library-tests/CapturedNodes/MethodCallTypeInferenceUsage.ql rename to javascript/ql/test/library-tests/LocalObjects/MethodCallTypeInferenceUsage.ql diff --git a/javascript/ql/test/library-tests/CapturedNodes/method-calls.js b/javascript/ql/test/library-tests/LocalObjects/method-calls.js similarity index 100% rename from javascript/ql/test/library-tests/CapturedNodes/method-calls.js rename to javascript/ql/test/library-tests/LocalObjects/method-calls.js diff --git a/javascript/ql/test/library-tests/CapturedNodes/tst.js b/javascript/ql/test/library-tests/LocalObjects/tst.js similarity index 100% rename from javascript/ql/test/library-tests/CapturedNodes/tst.js rename to javascript/ql/test/library-tests/LocalObjects/tst.js diff --git a/javascript/ql/test/library-tests/CapturedNodes/tst.ts b/javascript/ql/test/library-tests/LocalObjects/tst.ts similarity index 100% rename from javascript/ql/test/library-tests/CapturedNodes/tst.ts rename to javascript/ql/test/library-tests/LocalObjects/tst.ts From ab1b1c1431058c0c8dac45814704b1110c8fdafe Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 25 Feb 2019 16:11:35 +0100 Subject: [PATCH 25/71] JS: update docstring --- javascript/ql/src/semmle/javascript/dataflow/LocalObjects.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/ql/src/semmle/javascript/dataflow/LocalObjects.qll b/javascript/ql/src/semmle/javascript/dataflow/LocalObjects.qll index cf7575fcbd45..21e24c84c1c4 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/LocalObjects.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/LocalObjects.qll @@ -1,5 +1,5 @@ /** - * Provides classes for the nodes that the dataflow library can reason about soundly. + * Provides classes for the local objects that the dataflow library can reason about soundly. */ import javascript From 9511bdf6ae766416e808b293fb850246ee2bd9d2 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Tue, 26 Feb 2019 10:07:00 +0100 Subject: [PATCH 26/71] JS: address review comment --- .../ql/src/semmle/javascript/dataflow/LocalObjects.qll | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/dataflow/LocalObjects.qll b/javascript/ql/src/semmle/javascript/dataflow/LocalObjects.qll index 21e24c84c1c4..c8f1bcf0f0df 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/LocalObjects.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/LocalObjects.qll @@ -54,8 +54,10 @@ private predicate exposedAsReceiver(DataFlow::SourceNode n) { } /** - * A source that is entirely local, i.e. the dataflow library models all of its flow. - * All uses of the node are modeled by `this.flowsTo(_)` and related predicates. + * An object that is entirely local, in the sense that the dataflow + * library models all of its flow. + * + * All uses of this node are modeled by `this.flowsTo(_)` and related predicates. */ class LocalObject extends DataFlow::SourceNode { LocalObject() { From f8ae56a27cfd7e8896b5dd135bbb3021a518471d Mon Sep 17 00:00:00 2001 From: Raul Garcia Date: Tue, 26 Feb 2019 16:22:39 -0800 Subject: [PATCH 27/71] Improving documentation --- csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp index 1e71a53840ee..b72fa19140e3 100644 --- a/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp +++ b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp @@ -4,6 +4,11 @@

    Classes that implement System.Security.Cryptography.ICryptoTransform are not thread safe.

    +

    The root cause of the problem is because of the way these class are implemented using Microsoft CAPI/CNG patterns.

    +

    For example, a hash class implementing this interface, typically there would be an instance-specific hash object (i.e. using BCryptCreateHash function), which can be called multiple times to add data to the hash (i.e. BCryptHashData), and finally calling the function that would finish the hash & get back the data (i.e. BCryptFinishHash).

    +

    The implementation would potentially allow the same hash object to be called with data from multiple threads before calling the finish function, thus leading to potentially incorrect results.

    +

    Because of this pattern, you can expect that, if you have multiple threads hashing "abc" on a static hash object, you may occasionally obtain the results (incorrectly) for hashing "abcabc", or face other unexpected behavior.

    +

    It is very unlikely somebody outside Microsoft would write a class that implements ICryptoTransform, and even if they do, it is likely that they will follow the same common pattern than the existing classes implementing this interface.

    Any object that implements System.Security.Cryptography.ICryptoTransform should not be used in concurrent threads as the instance members of such object are also not thread safe.

    Potential problems may not be evident at first, but can range from explicit errors such as exceptions, to incorrect results when sharing an instance of such object in multiple threads.

    From 742c1d0fa7456b639576e1acf725e8ae285c7340 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 7 Feb 2019 11:08:27 +0000 Subject: [PATCH 28/71] Python: Add test skeleton for falcon web framework. --- .../library-tests/web/falcon/Routing.expected | 1 + .../ql/test/library-tests/web/falcon/Routing.ql | 7 +++++++ .../test/library-tests/web/falcon/Sinks.expected | 1 + python/ql/test/library-tests/web/falcon/Sinks.ql | 10 ++++++++++ .../library-tests/web/falcon/Sources.expected | 1 + .../ql/test/library-tests/web/falcon/Sources.ql | 10 ++++++++++ .../test/library-tests/web/falcon/Taint.expected | 1 + python/ql/test/library-tests/web/falcon/Taint.ql | 13 +++++++++++++ python/ql/test/library-tests/web/falcon/options | 2 ++ python/ql/test/library-tests/web/falcon/test.py | 16 ++++++++++++++++ .../query-tests/Security/lib/falcon/__init__.py | 4 ++++ .../test/query-tests/Security/lib/falcon/api.py | 14 ++++++++++++++ .../query-tests/Security/lib/falcon/request.py | 3 +++ .../query-tests/Security/lib/falcon/response.py | 4 ++++ 14 files changed, 87 insertions(+) create mode 100644 python/ql/test/library-tests/web/falcon/Routing.expected create mode 100644 python/ql/test/library-tests/web/falcon/Routing.ql create mode 100644 python/ql/test/library-tests/web/falcon/Sinks.expected create mode 100644 python/ql/test/library-tests/web/falcon/Sinks.ql create mode 100644 python/ql/test/library-tests/web/falcon/Sources.expected create mode 100644 python/ql/test/library-tests/web/falcon/Sources.ql create mode 100644 python/ql/test/library-tests/web/falcon/Taint.expected create mode 100644 python/ql/test/library-tests/web/falcon/Taint.ql create mode 100644 python/ql/test/library-tests/web/falcon/options create mode 100644 python/ql/test/library-tests/web/falcon/test.py create mode 100644 python/ql/test/query-tests/Security/lib/falcon/__init__.py create mode 100644 python/ql/test/query-tests/Security/lib/falcon/api.py create mode 100644 python/ql/test/query-tests/Security/lib/falcon/request.py create mode 100644 python/ql/test/query-tests/Security/lib/falcon/response.py diff --git a/python/ql/test/library-tests/web/falcon/Routing.expected b/python/ql/test/library-tests/web/falcon/Routing.expected new file mode 100644 index 000000000000..f45d4b694dab --- /dev/null +++ b/python/ql/test/library-tests/web/falcon/Routing.expected @@ -0,0 +1 @@ +fail \ No newline at end of file diff --git a/python/ql/test/library-tests/web/falcon/Routing.ql b/python/ql/test/library-tests/web/falcon/Routing.ql new file mode 100644 index 000000000000..3a10c78ddcba --- /dev/null +++ b/python/ql/test/library-tests/web/falcon/Routing.ql @@ -0,0 +1,7 @@ +import python + +import semmle.python.web.bottle.General + +from BottleRoute route + +select route.getUrl(), route.getFunction() diff --git a/python/ql/test/library-tests/web/falcon/Sinks.expected b/python/ql/test/library-tests/web/falcon/Sinks.expected new file mode 100644 index 000000000000..f45d4b694dab --- /dev/null +++ b/python/ql/test/library-tests/web/falcon/Sinks.expected @@ -0,0 +1 @@ +fail \ No newline at end of file diff --git a/python/ql/test/library-tests/web/falcon/Sinks.ql b/python/ql/test/library-tests/web/falcon/Sinks.ql new file mode 100644 index 000000000000..34aa1cfc429c --- /dev/null +++ b/python/ql/test/library-tests/web/falcon/Sinks.ql @@ -0,0 +1,10 @@ + +import python + +import semmle.python.web.HttpRequest +import semmle.python.web.HttpResponse +import semmle.python.security.strings.Untrusted + +from TaintSink sink, TaintKind kind +where sink.sinks(kind) +select sink.getLocation().toString(), sink.(ControlFlowNode).getNode().toString(), kind diff --git a/python/ql/test/library-tests/web/falcon/Sources.expected b/python/ql/test/library-tests/web/falcon/Sources.expected new file mode 100644 index 000000000000..f45d4b694dab --- /dev/null +++ b/python/ql/test/library-tests/web/falcon/Sources.expected @@ -0,0 +1 @@ +fail \ No newline at end of file diff --git a/python/ql/test/library-tests/web/falcon/Sources.ql b/python/ql/test/library-tests/web/falcon/Sources.ql new file mode 100644 index 000000000000..c1b9cc82e197 --- /dev/null +++ b/python/ql/test/library-tests/web/falcon/Sources.ql @@ -0,0 +1,10 @@ + +import python + +import semmle.python.web.HttpRequest +import semmle.python.security.strings.Untrusted + + +from TaintSource src, TaintKind kind +where src.isSourceOf(kind) and not kind.matches("tornado%") +select src.getLocation().toString(), src.(ControlFlowNode).getNode().toString(), kind diff --git a/python/ql/test/library-tests/web/falcon/Taint.expected b/python/ql/test/library-tests/web/falcon/Taint.expected new file mode 100644 index 000000000000..9dae856c349e --- /dev/null +++ b/python/ql/test/library-tests/web/falcon/Taint.expected @@ -0,0 +1 @@ +fail diff --git a/python/ql/test/library-tests/web/falcon/Taint.ql b/python/ql/test/library-tests/web/falcon/Taint.ql new file mode 100644 index 000000000000..6699c9bd9dae --- /dev/null +++ b/python/ql/test/library-tests/web/falcon/Taint.ql @@ -0,0 +1,13 @@ + +import python + + +import semmle.python.web.HttpRequest +import semmle.python.web.HttpResponse +import semmle.python.security.strings.Untrusted + + +from TaintedNode node + +select node.getLocation().toString(), node.getNode().getNode().toString(), node.getTaintKind() + diff --git a/python/ql/test/library-tests/web/falcon/options b/python/ql/test/library-tests/web/falcon/options new file mode 100644 index 000000000000..3eb8fa37213e --- /dev/null +++ b/python/ql/test/library-tests/web/falcon/options @@ -0,0 +1,2 @@ +semmle-extractor-options: --max-import-depth=3 --lang=3 -p ../../../query-tests/Security/lib/ +optimize: true diff --git a/python/ql/test/library-tests/web/falcon/test.py b/python/ql/test/library-tests/web/falcon/test.py new file mode 100644 index 000000000000..84817e9a969e --- /dev/null +++ b/python/ql/test/library-tests/web/falcon/test.py @@ -0,0 +1,16 @@ + + +from falcon import API + +app = API() + +class Handler(object): + + def on_get(self, req, resp): + ... + + def on_post(self, req, resp): + ... + +app.add_route('/hello', Handler()) + diff --git a/python/ql/test/query-tests/Security/lib/falcon/__init__.py b/python/ql/test/query-tests/Security/lib/falcon/__init__.py new file mode 100644 index 000000000000..5983e6a41a57 --- /dev/null +++ b/python/ql/test/query-tests/Security/lib/falcon/__init__.py @@ -0,0 +1,4 @@ + +from falcon.api import API +from falcon.request import Request +from falcon.response import Response diff --git a/python/ql/test/query-tests/Security/lib/falcon/api.py b/python/ql/test/query-tests/Security/lib/falcon/api.py new file mode 100644 index 000000000000..2763d8ca3e1b --- /dev/null +++ b/python/ql/test/query-tests/Security/lib/falcon/api.py @@ -0,0 +1,14 @@ + +"""Falcon API class.""" + +class API(object): + + def add_route(self, uri_template, resource, **kwargs): + pass + + def add_sink(self, sink, prefix=r'/'): + pass + + def add_error_handler(self, exception, handler=None): + pass + diff --git a/python/ql/test/query-tests/Security/lib/falcon/request.py b/python/ql/test/query-tests/Security/lib/falcon/request.py new file mode 100644 index 000000000000..4e39d52d8413 --- /dev/null +++ b/python/ql/test/query-tests/Security/lib/falcon/request.py @@ -0,0 +1,3 @@ + +class Request(object): + pass diff --git a/python/ql/test/query-tests/Security/lib/falcon/response.py b/python/ql/test/query-tests/Security/lib/falcon/response.py new file mode 100644 index 000000000000..d03bbee54ac9 --- /dev/null +++ b/python/ql/test/query-tests/Security/lib/falcon/response.py @@ -0,0 +1,4 @@ + + +class Response(object): + pass From 6a48420191f48c72ff57eda96b081df2e7b4f673 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 7 Feb 2019 14:46:11 +0000 Subject: [PATCH 29/71] Python: Basic support for falcon framework; routing and requests. --- change-notes/1.20/analysis-python.md | 1 + .../1.20/support/python-frameworks.csv | 1 + .../python/security/strings/External.qll | 20 ++++++ .../ql/src/semmle/python/web/HttpRequest.qll | 1 + .../src/semmle/python/web/falcon/General.qll | 68 +++++++++++++++++++ .../src/semmle/python/web/falcon/Request.qll | 55 +++++++++++++++ .../library-tests/web/falcon/Routing.expected | 3 +- .../test/library-tests/web/falcon/Routing.ql | 7 +- .../library-tests/web/falcon/Taint.expected | 15 +++- .../ql/test/library-tests/web/falcon/Taint.ql | 2 +- .../ql/test/library-tests/web/falcon/test.py | 13 +++- 11 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 python/ql/src/semmle/python/web/falcon/General.qll create mode 100644 python/ql/src/semmle/python/web/falcon/Request.qll diff --git a/change-notes/1.20/analysis-python.md b/change-notes/1.20/analysis-python.md index cc5442d8c76a..31b52a58a2a0 100644 --- a/change-notes/1.20/analysis-python.md +++ b/change-notes/1.20/analysis-python.md @@ -39,6 +39,7 @@ The API has been improved to declutter the global namespace and improve discover * Added support for the `dill` pickle library. * Added support for the `bottle` web framework. + * Added support for the `falcon` web API framework. * Added support for the `turbogears` web framework. diff --git a/change-notes/1.20/support/python-frameworks.csv b/change-notes/1.20/support/python-frameworks.csv index cafdbe20a83a..08a10d41ef9b 100644 --- a/change-notes/1.20/support/python-frameworks.csv +++ b/change-notes/1.20/support/python-frameworks.csv @@ -1,6 +1,7 @@ Name, Category Bottle, Web framework Django, Web application framework +Falcon, Web API framework Flask, Microframework Pyramid, Web application framework Tornado, Web application framework and asynchronous networking library diff --git a/python/ql/src/semmle/python/security/strings/External.qll b/python/ql/src/semmle/python/security/strings/External.qll index b79049fff0a4..cd024b063a74 100644 --- a/python/ql/src/semmle/python/security/strings/External.qll +++ b/python/ql/src/semmle/python/security/strings/External.qll @@ -96,3 +96,23 @@ private predicate json_load(ControlFlowNode fromnode, CallNode tonode) { ) } +/** A kind of "taint", representing am open file-like object from an external source. */ +class ExternalFileObject extends TaintKind { + + ExternalFileObject() { + this = "file[" + any(ExternalStringKind key) + "]" + } + + + /** Gets the taint kind for item in this sequence */ + TaintKind getValue() { + this = "file[" + result + "]" + } + + override TaintKind getTaintOfMethodResult(string name) { + name = "read" and result = this.getValue() + } + +} + + diff --git a/python/ql/src/semmle/python/web/HttpRequest.qll b/python/ql/src/semmle/python/web/HttpRequest.qll index bdd13d03ff96..066f6c1e27fd 100644 --- a/python/ql/src/semmle/python/web/HttpRequest.qll +++ b/python/ql/src/semmle/python/web/HttpRequest.qll @@ -5,3 +5,4 @@ import semmle.python.web.pyramid.Request import semmle.python.web.twisted.Request import semmle.python.web.bottle.Request import semmle.python.web.turbogears.Request +import semmle.python.web.falcon.Request diff --git a/python/ql/src/semmle/python/web/falcon/General.qll b/python/ql/src/semmle/python/web/falcon/General.qll new file mode 100644 index 000000000000..4f420e215d75 --- /dev/null +++ b/python/ql/src/semmle/python/web/falcon/General.qll @@ -0,0 +1,68 @@ +import python +import semmle.python.web.Http + + +/** The falcon API class */ +ClassObject theFalconAPIClass() { + result = ModuleObject::named("falcon").getAttribute("API") +} + + +/** Holds if `route` is routed to `resource` + */ +private predicate api_route(CallNode route_call, ControlFlowNode route, ClassObject resource) { + route_call.getFunction().(AttrNode).getObject("add_route").refersTo(_, theFalconAPIClass(), _) and + route_call.getArg(0) = route and + route_call.getArg(1).refersTo(_, resource, _) +} + +class FalconRoute extends ControlFlowNode { + + FalconRoute() { + api_route(this, _, _) + } + + string getUrl() { + exists(StrConst url | + api_route(this, url.getAFlowNode(), _) and + result = url.getText() + ) + } + + ClassObject getResourceClass() { + api_route(this, _, result) + } + + FalconHandlerFunction getHandlerFunction() { + result = this.getResourceClass().lookupAttribute(_).(FunctionObject).getFunction() + } + + FalconHandlerFunction getHandlerFunction(string method) { + result = this.getResourceClass().lookupAttribute("on_" + method).(FunctionObject).getFunction() + } + +} + +class FalconHandlerFunction extends Function { + + string method; + + FalconHandlerFunction() { + exists(ClassObject resource | + resource.lookupAttribute("on_" + method).(FunctionObject).getFunction() = this + ) + } + + string getMethod() { + result = method.toUpperCase() + } + + Parameter getRequest() { + result = this.getArg(1) + } + + Parameter getResponse() { + result = this.getArg(2) + } + +} diff --git a/python/ql/src/semmle/python/web/falcon/Request.qll b/python/ql/src/semmle/python/web/falcon/Request.qll new file mode 100644 index 000000000000..3cf9563d76f3 --- /dev/null +++ b/python/ql/src/semmle/python/web/falcon/Request.qll @@ -0,0 +1,55 @@ +import python + +import semmle.python.security.TaintTracking +import semmle.python.web.Http +import semmle.python.web.falcon.General +import semmle.python.security.strings.External + +/** https://falcon.readthedocs.io/en/stable/api/request_and_response.html */ +class FalconRequest extends TaintKind { + + FalconRequest() { + this = "falcon.request" + } + + override TaintKind getTaintOfAttribute(string name) { + // name = "env" and result instanceof WsgiEnvironment + result instanceof ExternalStringKind and + ( + name = "uri" or name = "url" or + name = "forwarded_uri" or + name = "relative_uri" or + name = "query_string" + ) + or + result instanceof ExternalStringDictKind and + ( + name = "cookies" or name = "params" + ) + or + name = "stream" and result instanceof ExternalFileObject + } + + override TaintKind getTaintOfMethodResult(string name) { + name = "get_param" and result instanceof ExternalStringKind + or + name = "get_param_as_json" and result instanceof ExternalJsonKind + or + name = "get_param_as_list" and result instanceof ExternalStringSequenceKind + } +} + +class FalconRequestParameter extends TaintSource { + + FalconRequestParameter() { + exists(FalconHandlerFunction f | + f.getRequest() = this.(ControlFlowNode).getNode() + ) + } + + override predicate isSourceOf(TaintKind k) { + k instanceof FalconRequest + } + +} + diff --git a/python/ql/test/library-tests/web/falcon/Routing.expected b/python/ql/test/library-tests/web/falcon/Routing.expected index f45d4b694dab..383937b03ffc 100644 --- a/python/ql/test/library-tests/web/falcon/Routing.expected +++ b/python/ql/test/library-tests/web/falcon/Routing.expected @@ -1 +1,2 @@ -fail \ No newline at end of file +| /hello | get | test.py:9:5:9:32 | Function on_get | +| /hello | post | test.py:12:5:12:33 | Function on_post | diff --git a/python/ql/test/library-tests/web/falcon/Routing.ql b/python/ql/test/library-tests/web/falcon/Routing.ql index 3a10c78ddcba..0596664ba760 100644 --- a/python/ql/test/library-tests/web/falcon/Routing.ql +++ b/python/ql/test/library-tests/web/falcon/Routing.ql @@ -1,7 +1,8 @@ import python -import semmle.python.web.bottle.General +import semmle.python.web.falcon.General -from BottleRoute route +from FalconRoute route, string method + +select route.getUrl(), method, route.getHandlerFunction(method) -select route.getUrl(), route.getFunction() diff --git a/python/ql/test/library-tests/web/falcon/Taint.expected b/python/ql/test/library-tests/web/falcon/Taint.expected index 9dae856c349e..c060b1e6c4b3 100644 --- a/python/ql/test/library-tests/web/falcon/Taint.expected +++ b/python/ql/test/library-tests/web/falcon/Taint.expected @@ -1 +1,14 @@ -fail +| test.py:9 | req | falcon.request | +| test.py:10 | Attribute | file[externally controlled string] | +| test.py:10 | Attribute() | externally controlled string | +| test.py:10 | req | falcon.request | +| test.py:11 | Attribute() | externally controlled string | +| test.py:11 | Attribute() | json[externally controlled string] | +| test.py:11 | raw_json | externally controlled string | +| test.py:13 | Dict | {externally controlled string} | +| test.py:13 | Dict | {json[externally controlled string]} | +| test.py:15 | result | externally controlled string | +| test.py:15 | result | json[externally controlled string] | +| test.py:17 | result | {externally controlled string} | +| test.py:17 | result | {json[externally controlled string]} | +| test.py:19 | req | falcon.request | diff --git a/python/ql/test/library-tests/web/falcon/Taint.ql b/python/ql/test/library-tests/web/falcon/Taint.ql index 6699c9bd9dae..8c0141db567a 100644 --- a/python/ql/test/library-tests/web/falcon/Taint.ql +++ b/python/ql/test/library-tests/web/falcon/Taint.ql @@ -8,6 +8,6 @@ import semmle.python.security.strings.Untrusted from TaintedNode node - +where node.getLocation().getFile().getName().matches("%falcon/test.py") select node.getLocation().toString(), node.getNode().getNode().toString(), node.getTaintKind() diff --git a/python/ql/test/library-tests/web/falcon/test.py b/python/ql/test/library-tests/web/falcon/test.py index 84817e9a969e..ff804b172a3e 100644 --- a/python/ql/test/library-tests/web/falcon/test.py +++ b/python/ql/test/library-tests/web/falcon/test.py @@ -1,4 +1,4 @@ - +import json from falcon import API @@ -7,10 +7,17 @@ class Handler(object): def on_get(self, req, resp): - ... + raw_json = req.stream.read() + result = json.loads(raw_json) + resp.status = 200 + result = { + 'status': 'success', + 'data': result + } + resp.body = json.dumps(result) def on_post(self, req, resp): - ... + pass app.add_route('/hello', Handler()) From 9e268d77d030c525de636758b4452d5dd332ed42 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 19 Feb 2019 11:53:57 +0000 Subject: [PATCH 30/71] Python: Add responses to Falcon framework support. --- .../ql/src/semmle/python/web/HttpResponse.qll | 1 + .../src/semmle/python/web/falcon/Response.qll | 48 +++++++++++++++++++ .../library-tests/web/falcon/Routing.expected | 2 +- .../library-tests/web/falcon/Sinks.expected | 2 +- .../library-tests/web/falcon/Sources.expected | 3 +- .../library-tests/web/falcon/Taint.expected | 4 ++ 6 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 python/ql/src/semmle/python/web/falcon/Response.qll diff --git a/python/ql/src/semmle/python/web/HttpResponse.qll b/python/ql/src/semmle/python/web/HttpResponse.qll index e5f1824434e0..3c5d45701a79 100644 --- a/python/ql/src/semmle/python/web/HttpResponse.qll +++ b/python/ql/src/semmle/python/web/HttpResponse.qll @@ -5,3 +5,4 @@ import semmle.python.web.tornado.Response import semmle.python.web.twisted.Response import semmle.python.web.bottle.Response import semmle.python.web.turbogears.Response +import semmle.python.web.falcon.Response diff --git a/python/ql/src/semmle/python/web/falcon/Response.qll b/python/ql/src/semmle/python/web/falcon/Response.qll new file mode 100644 index 000000000000..c55f164b1255 --- /dev/null +++ b/python/ql/src/semmle/python/web/falcon/Response.qll @@ -0,0 +1,48 @@ +import python + + +import semmle.python.security.TaintTracking +import semmle.python.web.Http +import semmle.python.web.falcon.General +import semmle.python.security.strings.External + + +/** https://falcon.readthedocs.io/en/stable/api/request_and_response.html */ +class FalconResponse extends TaintKind { + + FalconResponse() { + this = "falcon.response" + } + +} + +class FalconResponseParameter extends TaintSource { + + FalconResponseParameter() { + exists(FalconHandlerFunction f | + f.getResponse() = this.(ControlFlowNode).getNode() + ) + } + + override predicate isSourceOf(TaintKind k) { + k instanceof FalconResponse + } + +} + +class FalconResponseBodySink extends TaintSink { + + FalconResponseBodySink() { + exists(AttrNode attr | + any(FalconResponse f).taints(attr.getObject("body")) | + attr.(DefinitionNode).getValue() = this + ) + } + + override predicate sinks(TaintKind kind) { + kind instanceof ExternalStringKind + } + +} + + diff --git a/python/ql/test/library-tests/web/falcon/Routing.expected b/python/ql/test/library-tests/web/falcon/Routing.expected index 383937b03ffc..2756730b59c3 100644 --- a/python/ql/test/library-tests/web/falcon/Routing.expected +++ b/python/ql/test/library-tests/web/falcon/Routing.expected @@ -1,2 +1,2 @@ | /hello | get | test.py:9:5:9:32 | Function on_get | -| /hello | post | test.py:12:5:12:33 | Function on_post | +| /hello | post | test.py:19:5:19:33 | Function on_post | diff --git a/python/ql/test/library-tests/web/falcon/Sinks.expected b/python/ql/test/library-tests/web/falcon/Sinks.expected index f45d4b694dab..d11d1a5340e5 100644 --- a/python/ql/test/library-tests/web/falcon/Sinks.expected +++ b/python/ql/test/library-tests/web/falcon/Sinks.expected @@ -1 +1 @@ -fail \ No newline at end of file +| test.py:17 | Attribute() | externally controlled string | diff --git a/python/ql/test/library-tests/web/falcon/Sources.expected b/python/ql/test/library-tests/web/falcon/Sources.expected index f45d4b694dab..69ea9da4259d 100644 --- a/python/ql/test/library-tests/web/falcon/Sources.expected +++ b/python/ql/test/library-tests/web/falcon/Sources.expected @@ -1 +1,2 @@ -fail \ No newline at end of file +| test.py:9 | req | falcon.request | +| test.py:19 | req | falcon.request | diff --git a/python/ql/test/library-tests/web/falcon/Taint.expected b/python/ql/test/library-tests/web/falcon/Taint.expected index c060b1e6c4b3..0c7c2d991189 100644 --- a/python/ql/test/library-tests/web/falcon/Taint.expected +++ b/python/ql/test/library-tests/web/falcon/Taint.expected @@ -1,14 +1,18 @@ | test.py:9 | req | falcon.request | +| test.py:9 | resp | falcon.response | | test.py:10 | Attribute | file[externally controlled string] | | test.py:10 | Attribute() | externally controlled string | | test.py:10 | req | falcon.request | | test.py:11 | Attribute() | externally controlled string | | test.py:11 | Attribute() | json[externally controlled string] | | test.py:11 | raw_json | externally controlled string | +| test.py:12 | resp | falcon.response | | test.py:13 | Dict | {externally controlled string} | | test.py:13 | Dict | {json[externally controlled string]} | | test.py:15 | result | externally controlled string | | test.py:15 | result | json[externally controlled string] | +| test.py:17 | resp | falcon.response | | test.py:17 | result | {externally controlled string} | | test.py:17 | result | {json[externally controlled string]} | | test.py:19 | req | falcon.request | +| test.py:19 | resp | falcon.response | From 2ed37903d87765adfdab7937f0e69ca2dbac6d52 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Wed, 27 Feb 2019 11:54:59 +0000 Subject: [PATCH 31/71] JavaScript: Include list of relevant environment variables in Javadoc for `AutoBuild`. --- .../src/com/semmle/js/extractor/AutoBuild.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index 42b969f191ee..762df7fe8e58 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -58,6 +58,23 @@ * * *

    + * Additionally, the following environment variables may be set to customise extraction + * (explained in more detail below): + *

    + * + *
      + *
    • LGTM_INDEX_INCLUDE: a newline-separated list of paths to include
      • + *
    • LGTM_INDEX_EXCLUDE: a newline-separated list of paths to exclude
      • + *
    • LGTM_REPOSITORY_FOLDERS_CSV: the path of a CSV file containing file classifications
      • + *
    • LGTM_INDEX_FILTERS: a newline-separated list of {@link ProjectLayout}-style + * patterns that can be used to refine the list of files to include and exclude
      • + *
    • LGTM_INDEX_TYPESCRIPT: whether to extract TypeScript
      • + *
    • LGTM_INDEX_THREADS: the maximum number of files to extract in parallel
      • + *
    • LGTM_TRAP_CACHE: the path of a directory to use for trap caching
      • + *
    • LGTM_TRAP_CACHE_BOUND: the size to bound the trap cache to
      • +
    + * + *

    * It extracts the following: *

    * From 9d77619afc9364957dc69a1480a97b30f7e5bb2e Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Wed, 27 Feb 2019 12:02:01 +0000 Subject: [PATCH 32/71] JavaScript: Make file types customisable in AutoBuild. Every once in a while we encounter projects using some custom file extension for files that we could in principle extract, but since the extractor doesn't know about the extension the files are skipped. To handle this, the legacy extractor has a `--file-type` option that one can use to specify a file type to use for all files in that particular extraction. So far, `AutoBuild` has nothing of the sort. This PR proposes to introduce an environment variable `LGTM_INDEX_FILETYPES` to allow a similar customisation. In the fullness of time, this variable would be set through `lgtm.yml` in the usual way, but for now it is undocumented and for internal use only. Specifically, `LGTM_INDEX_FILETYPES` is a newline-separated list of ".extension:filetype" pairs, specifying that files with the given `.extension` should be extracted as type `filetype`, where `filetype` is one of `js`, `html`, `json`, `typescript` or `yaml`. For example, `.jsm:js` causes all `.jsm` files to be extracted as JavaScript. This can also be used to override default file types: for example, by specifying `.js:typescript` all JavaScript files will be extracted as TypeScript. --- .../com/semmle/js/extractor/AutoBuild.java | 82 +++++++++++++++++-- .../js/extractor/test/AutoBuildTests.java | 61 ++++++++++++-- 2 files changed, 128 insertions(+), 15 deletions(-) diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index 762df7fe8e58..71bb515ec54c 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -16,8 +16,10 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -69,6 +71,8 @@ *
  • LGTM_INDEX_FILTERS: a newline-separated list of {@link ProjectLayout}-style * patterns that can be used to refine the list of files to include and exclude
    • *
  • LGTM_INDEX_TYPESCRIPT: whether to extract TypeScript
    • + *
  • LGTM_INDEX_FILETYPES: a newline-separated list of ".extension:filetype" pairs + * specifying which {@link FileType} to use for the given extension
    • *
  • LGTM_INDEX_THREADS: the maximum number of files to extract in parallel
    • *
  • LGTM_TRAP_CACHE: the path of a directory to use for trap caching
    • *
  • LGTM_TRAP_CACHE_BOUND: the size to bound the trap cache to
    • @@ -160,6 +164,12 @@ *

    * *

    + * The environment variable LGTM_INDEX_FILETYPES may be set to a newline-separated + * list of file type specifications of the form .extension:filetype, causing all + * files whose name ends in .extension to also be included by default. + *

    + * + *

    * The default exclusion patterns cause the following files to be excluded: *

    *
      @@ -174,6 +184,11 @@ *

      * *

      + * The file type as which a file is extracted can be customised via the LGTM_INDEX_FILETYPES + * environment variable explained above. + *

      + * + *

      * Note that all these customisations only apply to LGTM_SRC. Extraction of * externs is not customisable. *

      @@ -193,6 +208,7 @@ public class AutoBuild { private final ExtractorOutputConfig outputConfig; private final ITrapCache trapCache; + private final Map fileTypes = new LinkedHashMap<>(); private final Set includes = new LinkedHashSet<>(); private final Set excludes = new LinkedHashSet<>(); private ProjectLayout filters; @@ -208,6 +224,7 @@ public AutoBuild() { this.trapCache = mkTrapCache(); this.typeScriptMode = getEnumFromEnvVar("LGTM_INDEX_TYPESCRIPT", TypeScriptMode.class, TypeScriptMode.BASIC); this.defaultEncoding = getEnvVar("LGTM_INDEX_DEFAULT_ENCODING"); + setupFileTypes(); setupMatchers(); } @@ -277,6 +294,25 @@ private ITrapCache mkTrapCache() { return trapCache; } + private void setupFileTypes() { + for (String spec : Main.NEWLINE.split(getEnvVar("LGTM_INDEX_FILETYPES", ""))) { + spec = spec.trim(); + if (spec.isEmpty()) + continue; + String[] fields = spec.split(":"); + if (fields.length != 2) + continue; + String extension = fields[0].trim(); + String fileType = fields[1].trim(); + try { + fileTypes.put(extension, FileType.valueOf(StringUtil.uc(fileType))); + } catch (IllegalArgumentException e) { + Exceptions.ignore(e, "We construct a better error message."); + throw new UserError("Invalid file type '" + fileType + "'."); + } + } + } + /** * Set up include and exclude matchers based on environment variables. */ @@ -350,6 +386,10 @@ private void setupFilters() { patterns.add("**/.eslintrc*"); patterns.add("**/package.json"); + // include any explicitly specified extensions + for (String extension : fileTypes.keySet()) + patterns.add("**/*" + extension); + // exclude files whose name strongly suggests they are minified patterns.add("-**/*.min.js"); patterns.add("-**/*-min.js"); @@ -483,28 +523,48 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO * Extract all supported candidate files that pass the filters. */ private void extractSource() throws IOException { - ExtractorConfig config = new ExtractorConfig(true); - config = config.withSourceType(getSourceType()); - config = config.withTypeScriptMode(typeScriptMode); - if (defaultEncoding != null) - config = config.withDefaultEncoding(defaultEncoding); - FileExtractor extractor = new FileExtractor(config, outputConfig, trapCache); + // default extractor + FileExtractor defaultExtractor = new FileExtractor(mkExtractorConfig(), outputConfig, trapCache); + + // custom extractor for explicitly specified file types + Map customExtractors = new LinkedHashMap<>(); + for (Map.Entry spec : fileTypes.entrySet()) { + String extension = spec.getKey(); + String fileType = spec.getValue().name(); + ExtractorConfig extractorConfig = mkExtractorConfig().withFileType(fileType); + customExtractors.put(extension, new FileExtractor(extractorConfig, outputConfig, trapCache)); + } Set filesToExtract = new LinkedHashSet<>(); List tsconfigFiles = new ArrayList<>(); - findFilesToExtract(extractor, filesToExtract, tsconfigFiles); + findFilesToExtract(defaultExtractor, filesToExtract, tsconfigFiles); // extract TypeScript projects and files - Set extractedFiles = extractTypeScript(extractor, filesToExtract, tsconfigFiles); + Set extractedFiles = extractTypeScript(defaultExtractor, filesToExtract, tsconfigFiles); // extract remaining files for (Path f : filesToExtract) { if (extractedFiles.add(f)) { + FileExtractor extractor = defaultExtractor; + if (!fileTypes.isEmpty()) { + String extension = FileUtil.extension(f); + if (customExtractors.containsKey(extension)) + extractor = customExtractors.get(extension); + } extract(extractor, f, null); } } } + private ExtractorConfig mkExtractorConfig() { + ExtractorConfig config = new ExtractorConfig(true); + config = config.withSourceType(getSourceType()); + config = config.withTypeScriptMode(typeScriptMode); + if (defaultEncoding != null) + config = config.withDefaultEncoding(defaultEncoding); + return config; + } + private Set extractTypeScript(FileExtractor extractor, Set files, List tsconfig) { Set extractedFiles = new LinkedHashSet<>(); @@ -591,7 +651,11 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO return FileVisitResult.SKIP_SUBTREE; // extract files that are supported and pass the include/exclude patterns - if (extractor.supports(file.toFile()) && isFileIncluded(file)) { + boolean supported = extractor.supports(file.toFile()); + if (!supported && !fileTypes.isEmpty()) { + supported = fileTypes.containsKey(FileUtil.extension(file)); + } + if (supported && isFileIncluded(file)) { filesToExtract.add(normalizePath(file)); } diff --git a/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java b/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java index c2c156ee003c..77b3e5119604 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java +++ b/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java @@ -23,6 +23,7 @@ import com.semmle.js.extractor.AutoBuild; import com.semmle.js.extractor.ExtractorState; import com.semmle.js.extractor.FileExtractor; +import com.semmle.js.extractor.FileExtractor.FileType; import com.semmle.util.data.StringUtil; import com.semmle.util.exception.UserError; import com.semmle.util.files.FileUtil8; @@ -74,15 +75,31 @@ public void teardown() throws IOException { /** * Add a file under {@code root} that we either do or don't expect to be extracted, * depending on the value of {@code extracted}. If the file is expected to be - * extracted, its path is added to {@link #expected}. + * extracted, its path is added to {@link #expected}. If non-null, parameter + * {@code fileType} indicates the file type with which we expect the file to be extracted. + */ + private Path addFile(boolean extracted, FileType fileType, Path root, String... components) throws IOException { + Path f = addFile(root, components); + if (extracted) { + expected.add(f + (fileType == null ? "" : ":" + fileType.toString())); + } + return f; + } + + /** + * Add a file with default file type; see {@link #addFile(boolean, FileType, Path, String...)}. */ private Path addFile(boolean extracted, Path root, String... components) throws IOException { + return addFile(extracted, null, root, components); + } + + /** + * Create a file at the specified path under {@code root} and return it. + */ + private Path addFile(Path root, String... components) throws IOException { Path p = Paths.get(root.toString(), components); Files.createDirectories(p.getParent()); - Path f = Files.createFile(p); - if (extracted) - expected.add(f.toString()); - return f; + return Files.createFile(p); } /** @@ -96,7 +113,10 @@ private void runTest() throws IOException { new AutoBuild() { @Override protected void extract(FileExtractor extractor, Path file, ExtractorState state) { - actual.add(file.toString()); + String extracted = file.toString(); + if (extractor.getConfig().hasFileType()) + extracted += ":" + extractor.getFileType(file.toFile()); + actual.add(extracted); } @Override @@ -453,4 +473,33 @@ public void minifiedFilesCanBeReIncluded() throws IOException { addFile(true, LGTM_SRC, "compute_min.js"); runTest(); } + + @Test + public void customExtensions() throws IOException { + envVars.put("LGTM_INDEX_FILETYPES", ".jsm:js\n.soy:html"); + addFile(true, FileType.JS, LGTM_SRC, "tst.jsm"); + addFile(false, LGTM_SRC, "tstjsm"); + addFile(true, FileType.HTML, LGTM_SRC, "tst.soy"); + addFile(true, LGTM_SRC, "tst.html"); + addFile(true, LGTM_SRC, "tst.js"); + runTest(); + } + + @Test + public void overrideExtension() throws IOException { + envVars.put("LGTM_INDEX_FILETYPES", ".js:typescript"); + addFile(true, FileType.TYPESCRIPT, LGTM_SRC, "tst.js"); + runTest(); + } + + @Test + public void invalidFileType() throws IOException { + envVars.put("LGTM_INDEX_FILETYPES", ".jsm:javascript"); + try { + runTest(); + Assert.fail("expected UserError"); + } catch (UserError ue) { + Assert.assertEquals("Invalid file type 'javascript'.", ue.getMessage()); + } + } } From b6648def1975ab57ef51447a2ab769a6ad806ab1 Mon Sep 17 00:00:00 2001 From: Asger F Date: Tue, 26 Feb 2019 12:18:55 +0000 Subject: [PATCH 33/71] JS: Add ClassNode.getAReceiverNode --- javascript/ql/src/semmle/javascript/dataflow/Nodes.qll | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/javascript/ql/src/semmle/javascript/dataflow/Nodes.qll b/javascript/ql/src/semmle/javascript/dataflow/Nodes.qll index 64cbe8bc5576..0d841e3a9650 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/Nodes.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/Nodes.qll @@ -650,6 +650,15 @@ class ClassNode extends DataFlow::SourceNode { .(AbstractCallable) .getFunction() } + + /** + * Gets the receiver of an instance member or constructor of this class. + */ + DataFlow::SourceNode getAReceiverNode() { + result = getConstructor().getReceiver() + or + result = getAnInstanceMember().getReceiver() + } } module ClassNode { From 9497199cbdeed990c74f9be8740e430f976a2990 Mon Sep 17 00:00:00 2001 From: Asger F Date: Tue, 26 Feb 2019 12:24:05 +0000 Subject: [PATCH 34/71] JS: add localFieldStep --- .../semmle/javascript/dataflow/DataFlow.qll | 10 ++++++++++ .../library-tests/ClassNode/FieldStep.expected | 3 +++ .../test/library-tests/ClassNode/FieldStep.ql | 5 +++++ .../ClassNode/InstanceMember.expected | 2 ++ .../ClassNode/InstanceMethod.expected | 1 + .../ClassNode/getAReceiverNode.expected | 18 ++++++++++++++++++ .../ClassNode/getAReceiverNode.ql | 4 ++++ .../ql/test/library-tests/ClassNode/tst2.js | 14 ++++++++++++++ 8 files changed, 57 insertions(+) create mode 100644 javascript/ql/test/library-tests/ClassNode/FieldStep.expected create mode 100644 javascript/ql/test/library-tests/ClassNode/FieldStep.ql create mode 100644 javascript/ql/test/library-tests/ClassNode/getAReceiverNode.expected create mode 100644 javascript/ql/test/library-tests/ClassNode/getAReceiverNode.ql create mode 100644 javascript/ql/test/library-tests/ClassNode/tst2.js diff --git a/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll b/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll index 87feadd43610..b58e85e838af 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll @@ -1078,6 +1078,16 @@ module DataFlow { ) } + /** + * Holds if there is a step from `pred` to `succ` through a field accessed through `this` in a class. + */ + predicate localFieldStep(DataFlow::Node pred, DataFlow::Node succ) { + exists (ClassNode cls, string prop | + pred = cls.getAReceiverNode().getAPropertyWrite(prop).getRhs() and + succ = cls.getAReceiverNode().getAPropertyRead(prop) + ) + } + /** * Gets the data flow node representing the source of definition `def`, taking * flow through IIFE calls into account. diff --git a/javascript/ql/test/library-tests/ClassNode/FieldStep.expected b/javascript/ql/test/library-tests/ClassNode/FieldStep.expected new file mode 100644 index 000000000000..23feb07fbfaf --- /dev/null +++ b/javascript/ql/test/library-tests/ClassNode/FieldStep.expected @@ -0,0 +1,3 @@ +| tst2.js:3:14:3:14 | x | tst2.js:7:5:7:10 | this.x | +| tst2.js:3:14:3:14 | x | tst2.js:8:25:8:30 | this.x | +| tst2.js:3:14:3:14 | x | tst2.js:12:12:12:17 | this.x | diff --git a/javascript/ql/test/library-tests/ClassNode/FieldStep.ql b/javascript/ql/test/library-tests/ClassNode/FieldStep.ql new file mode 100644 index 000000000000..75dbd42e4f80 --- /dev/null +++ b/javascript/ql/test/library-tests/ClassNode/FieldStep.ql @@ -0,0 +1,5 @@ +import javascript + +from DataFlow::Node pred, DataFlow::Node succ +where DataFlow::localFieldStep(pred, succ) +select pred, succ diff --git a/javascript/ql/test/library-tests/ClassNode/InstanceMember.expected b/javascript/ql/test/library-tests/ClassNode/InstanceMember.expected index d9d9bfdda342..1566c4ff9c20 100644 --- a/javascript/ql/test/library-tests/ClassNode/InstanceMember.expected +++ b/javascript/ql/test/library-tests/ClassNode/InstanceMember.expected @@ -1,4 +1,6 @@ | namespace.js:5:32:5:44 | function() {} | Baz.method | method | +| tst2.js:6:9:9:3 | () {\\n ... .x;\\n } | C.method | method | +| tst2.js:11:13:13:3 | () {\\n ... .x;\\n } | C.getter | getter | | tst.js:4:17:4:21 | () {} | A.instanceMethod | method | | tst.js:7:6:7:10 | () {} | A.bar | method | | tst.js:9:10:9:14 | () {} | A.baz | getter | diff --git a/javascript/ql/test/library-tests/ClassNode/InstanceMethod.expected b/javascript/ql/test/library-tests/ClassNode/InstanceMethod.expected index ac97e60da684..d0406361e6bb 100644 --- a/javascript/ql/test/library-tests/ClassNode/InstanceMethod.expected +++ b/javascript/ql/test/library-tests/ClassNode/InstanceMethod.expected @@ -1,4 +1,5 @@ | namespace.js:5:32:5:44 | function() {} | Baz.method | +| tst2.js:6:9:9:3 | () {\\n ... .x;\\n } | C.method | | tst.js:4:17:4:21 | () {} | A.instanceMethod | | tst.js:7:6:7:10 | () {} | A.bar | | tst.js:17:19:17:31 | function() {} | B.foo | diff --git a/javascript/ql/test/library-tests/ClassNode/getAReceiverNode.expected b/javascript/ql/test/library-tests/ClassNode/getAReceiverNode.expected new file mode 100644 index 000000000000..c17b1db03a4d --- /dev/null +++ b/javascript/ql/test/library-tests/ClassNode/getAReceiverNode.expected @@ -0,0 +1,18 @@ +| namespace.js:3:15:3:31 | function Baz() {} | namespace.js:3:15:3:14 | this | +| namespace.js:3:15:3:31 | function Baz() {} | namespace.js:5:32:5:31 | this | +| tst2.js:1:1:14:1 | class C ... ;\\n }\\n} | tst2.js:2:14:2:13 | this | +| tst2.js:1:1:14:1 | class C ... ;\\n }\\n} | tst2.js:6:9:6:8 | this | +| tst2.js:1:1:14:1 | class C ... ;\\n }\\n} | tst2.js:11:13:11:12 | this | +| tst.js:3:1:10:1 | class A ... () {}\\n} | tst.js:3:9:3:8 | this | +| tst.js:3:1:10:1 | class A ... () {}\\n} | tst.js:4:17:4:16 | this | +| tst.js:3:1:10:1 | class A ... () {}\\n} | tst.js:7:6:7:5 | this | +| tst.js:3:1:10:1 | class A ... () {}\\n} | tst.js:9:10:9:9 | this | +| tst.js:13:1:13:21 | class A ... ds A {} | tst.js:13:20:13:19 | this | +| tst.js:15:1:15:15 | function B() {} | tst.js:15:1:15:0 | this | +| tst.js:15:1:15:15 | function B() {} | tst.js:17:19:17:18 | this | +| tst.js:19:1:19:15 | function C() {} | tst.js:19:1:19:0 | this | +| tst.js:19:1:19:15 | function C() {} | tst.js:21:19:21:18 | this | +| tst.js:23:1:23:15 | function D() {} | tst.js:23:1:23:0 | this | +| tst.js:23:1:23:15 | function D() {} | tst.js:25:13:25:12 | this | +| tst.js:23:1:23:15 | function D() {} | tst.js:26:13:26:12 | this | +| tst.js:23:1:23:15 | function D() {} | tst.js:27:4:27:3 | this | diff --git a/javascript/ql/test/library-tests/ClassNode/getAReceiverNode.ql b/javascript/ql/test/library-tests/ClassNode/getAReceiverNode.ql new file mode 100644 index 000000000000..97428ff54c74 --- /dev/null +++ b/javascript/ql/test/library-tests/ClassNode/getAReceiverNode.ql @@ -0,0 +1,4 @@ +import javascript + +from DataFlow::ClassNode cls +select cls, cls.getAReceiverNode() diff --git a/javascript/ql/test/library-tests/ClassNode/tst2.js b/javascript/ql/test/library-tests/ClassNode/tst2.js new file mode 100644 index 000000000000..88746d8fcd40 --- /dev/null +++ b/javascript/ql/test/library-tests/ClassNode/tst2.js @@ -0,0 +1,14 @@ +class C { + constructor(x) { + this.x = x; + } + + method() { + this.x; + let closure = () => this.x; + } + + get getter() { + return this.x; + } +} From 9170d8515514cbd51d30a836e4be9da9020f9eaf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 25 Feb 2019 17:58:13 +0000 Subject: [PATCH 35/71] Python: Fix falcon sources to only be source if a route is attached. --- .../src/semmle/python/web/falcon/General.qll | 22 +++++++++---------- .../src/semmle/python/web/falcon/Response.qll | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/python/ql/src/semmle/python/web/falcon/General.qll b/python/ql/src/semmle/python/web/falcon/General.qll index 4f420e215d75..d74336f03cea 100644 --- a/python/ql/src/semmle/python/web/falcon/General.qll +++ b/python/ql/src/semmle/python/web/falcon/General.qll @@ -16,6 +16,10 @@ private predicate api_route(CallNode route_call, ControlFlowNode route, ClassObj route_call.getArg(1).refersTo(_, resource, _) } +private predicate route(FalconRoute route, Function target, string funcname) { + route.getResourceClass().lookupAttribute("on_" + funcname).(FunctionObject).getFunction() = target +} + class FalconRoute extends ControlFlowNode { FalconRoute() { @@ -33,28 +37,24 @@ class FalconRoute extends ControlFlowNode { api_route(this, _, result) } - FalconHandlerFunction getHandlerFunction() { - result = this.getResourceClass().lookupAttribute(_).(FunctionObject).getFunction() - } - FalconHandlerFunction getHandlerFunction(string method) { - result = this.getResourceClass().lookupAttribute("on_" + method).(FunctionObject).getFunction() + route(this, result, method) } } class FalconHandlerFunction extends Function { - string method; - FalconHandlerFunction() { - exists(ClassObject resource | - resource.lookupAttribute("on_" + method).(FunctionObject).getFunction() = this - ) + route(_, this, _) + } + + private string methodName() { + route(_, this, result) } string getMethod() { - result = method.toUpperCase() + result = this.methodName().toUpperCase() } Parameter getRequest() { diff --git a/python/ql/src/semmle/python/web/falcon/Response.qll b/python/ql/src/semmle/python/web/falcon/Response.qll index c55f164b1255..9e18d49138fa 100644 --- a/python/ql/src/semmle/python/web/falcon/Response.qll +++ b/python/ql/src/semmle/python/web/falcon/Response.qll @@ -40,7 +40,7 @@ class FalconResponseBodySink extends TaintSink { } override predicate sinks(TaintKind kind) { - kind instanceof ExternalStringKind + kind instanceof StringKind } } From 1ae18974d89f95408c76b75efc7c9ffaeed781fb Mon Sep 17 00:00:00 2001 From: Raul Garcia Date: Wed, 27 Feb 2019 18:41:23 -0800 Subject: [PATCH 36/71] Fixing bugs found during Code Review. --- .gitignore | 3 +++ .../ThreadUnSafeICryptoTransformBad.cs | 11 ++++------- .../ThreadUnSafeICryptoTransformGood.cs | 11 +++++++---- .../ThreadUnsafeICryptoTransform.qhelp | 17 ++++++++--------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 7e82b2f488ca..c614f8f91592 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ /.vs/ql/v15/Browse.VC.db /.vs/ProjectSettings.json +/.vs/ql_ICryptoTransform/v15/Browse.VC.opendb +/.vs/ql_ICryptoTransform/v15/Browse.VC.db +/.vs/ql_ICryptoTransform/v15/.suo diff --git a/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformBad.cs b/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformBad.cs index dbbc3586f981..b698abdb600d 100644 --- a/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformBad.cs +++ b/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformBad.cs @@ -1,9 +1,6 @@ -internal class TokenCacheThreadUnsafeICryptoTransformDemoFixed +internal class TokenCacheThreadUnsafeICryptoTransformDemo { - // We are replacing the static SHA256 field with an instance one - // - //private static SHA256 _sha = SHA256.Create(); - private SHA256 _sha = SHA256.Create(); + private static SHA256 _sha = SHA256.Create(); public string ComputeHash(string data) { @@ -21,8 +18,8 @@ static void Main(string[] args) Action action = (object obj) => { - var safeObj = new TokenCacheThreadUnsafeICryptoTransformDemoFixed(); - if (safeObj.ComputeHash((string)obj) != "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=") + var unsafeObj = new TokenCacheThreadUnsafeICryptoTransformDemo(); + if (unsafeObj.ComputeHash((string)obj) != "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=") { Console.WriteLine("**** We got incorrect Results!!! ****"); } diff --git a/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformGood.cs b/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformGood.cs index b698abdb600d..dbbc3586f981 100644 --- a/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformGood.cs +++ b/csharp/ql/src/Likely Bugs/ThreadUnSafeICryptoTransformGood.cs @@ -1,6 +1,9 @@ -internal class TokenCacheThreadUnsafeICryptoTransformDemo +internal class TokenCacheThreadUnsafeICryptoTransformDemoFixed { - private static SHA256 _sha = SHA256.Create(); + // We are replacing the static SHA256 field with an instance one + // + //private static SHA256 _sha = SHA256.Create(); + private SHA256 _sha = SHA256.Create(); public string ComputeHash(string data) { @@ -18,8 +21,8 @@ static void Main(string[] args) Action action = (object obj) => { - var unsafeObj = new TokenCacheThreadUnsafeICryptoTransformDemo(); - if (unsafeObj.ComputeHash((string)obj) != "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=") + var safeObj = new TokenCacheThreadUnsafeICryptoTransformDemoFixed(); + if (safeObj.ComputeHash((string)obj) != "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=") { Console.WriteLine("**** We got incorrect Results!!! ****"); } diff --git a/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp index b72fa19140e3..23046f5217a4 100644 --- a/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp +++ b/csharp/ql/src/Likely Bugs/ThreadUnsafeICryptoTransform.qhelp @@ -4,23 +4,22 @@

      Classes that implement System.Security.Cryptography.ICryptoTransform are not thread safe.

      -

      The root cause of the problem is because of the way these class are implemented using Microsoft CAPI/CNG patterns.

      -

      For example, a hash class implementing this interface, typically there would be an instance-specific hash object (i.e. using BCryptCreateHash function), which can be called multiple times to add data to the hash (i.e. BCryptHashData), and finally calling the function that would finish the hash & get back the data (i.e. BCryptFinishHash).

      -

      The implementation would potentially allow the same hash object to be called with data from multiple threads before calling the finish function, thus leading to potentially incorrect results.

      -

      Because of this pattern, you can expect that, if you have multiple threads hashing "abc" on a static hash object, you may occasionally obtain the results (incorrectly) for hashing "abcabc", or face other unexpected behavior.

      -

      It is very unlikely somebody outside Microsoft would write a class that implements ICryptoTransform, and even if they do, it is likely that they will follow the same common pattern than the existing classes implementing this interface.

      +

      This problem is caused by the way these classes are implemented using Microsoft CAPI/CNG patterns.

      +

      For example, when a hash class implements this interface, there would typically be an instance-specific hash object created (for example using BCryptCreateHash function). This object can be called multiple times to add data to the hash (for example BCryptHashData). Finally, a function is called that finishes the hash and returns the data (for example BCryptFinishHash).

      +

      Allowing the same hash object to be called with data from multiple threads before calling the finish function could potentially lead to incorrect results.

      +

      For example, if you have multiple threads hashing "abc" on a static hash object, you may occasionally obtain the results (incorrectly) for hashing "abcabc", or face other unexpected behavior.

      +

      It is very unlikely somebody outside Microsoft would write a class that implements ICryptoTransform, and even if they do, it is likely that they will follow the same common pattern as the existing classes implementing this interface.

      Any object that implements System.Security.Cryptography.ICryptoTransform should not be used in concurrent threads as the instance members of such object are also not thread safe.

      -

      Potential problems may not be evident at first, but can range from explicit errors such as exceptions, to incorrect results when sharing an instance of such object in multiple threads.

      +

      Potential problems may not be evident at first, but can range from explicit errors such as exceptions, to incorrect results when sharing an instance of such an object in multiple threads.

       -

      Verify that the object is not being shared across threads.

      -

      If it is shared accross instances. Consider changing the code to use a non-static object of type System.Security.Cryptography.ICryptoTransform instead.

      +

      If the object is shared across instances, you should consider changing the code to use a non-static object of type System.Security.Cryptography.ICryptoTransform instead.

      As an alternative, you could also look into using ThreadStatic attribute, but make sure you read the initialization remarks on the documentation.

       -

      This example demonstrates the dangers of using a static System.Security.Cryptography.ICryptoTransform in such a way that the results may be incorrect.

      +

      This example demonstrates the dangers of using a static System.Security.Cryptography.ICryptoTransform in a way that generates incorrect results.

      A simple fix is to change the _sha field from being a static member to an instance one by removing the static keyword.

      From e24ca8ec40407ecdb4ec0c7cfefe85503f313d77 Mon Sep 17 00:00:00 2001 From: Raul Garcia <42392023+raulgarciamsft@users.noreply.github.com> Date: Wed, 27 Feb 2019 18:43:33 -0800 Subject: [PATCH 37/71] Update .gitignore --- .gitignore | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitignore b/.gitignore index c614f8f91592..4b055e55a091 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,3 @@ /.vs/ql/v15/Browse.VC.opendb /.vs/ql/v15/Browse.VC.db /.vs/ProjectSettings.json - -/.vs/ql_ICryptoTransform/v15/Browse.VC.opendb -/.vs/ql_ICryptoTransform/v15/Browse.VC.db -/.vs/ql_ICryptoTransform/v15/.suo From 9eca21cb5a67443856ce3fb03d9bafd31a3e4ed2 Mon Sep 17 00:00:00 2001 From: Raul Garcia <42392023+raulgarciamsft@users.noreply.github.com> Date: Wed, 27 Feb 2019 18:43:51 -0800 Subject: [PATCH 38/71] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4b055e55a091..7e82b2f488ca 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ /.vs/ql/v15/Browse.VC.opendb /.vs/ql/v15/Browse.VC.db /.vs/ProjectSettings.json + From 8e8085ea1f80890a7282a6a932a780f7a4a05b5c Mon Sep 17 00:00:00 2001 From: Asger F Date: Thu, 28 Feb 2019 10:09:36 +0000 Subject: [PATCH 39/71] JS: add test --- .../Security/CWE-020/IncorrectSuffixCheck.expected | 1 + javascript/ql/test/query-tests/Security/CWE-020/tst.js | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/javascript/ql/test/query-tests/Security/CWE-020/IncorrectSuffixCheck.expected b/javascript/ql/test/query-tests/Security/CWE-020/IncorrectSuffixCheck.expected index 3c44bc286379..92e2f5968b87 100644 --- a/javascript/ql/test/query-tests/Security/CWE-020/IncorrectSuffixCheck.expected +++ b/javascript/ql/test/query-tests/Security/CWE-020/IncorrectSuffixCheck.expected @@ -8,3 +8,4 @@ | tst.js:55:32:55:71 | x.index ... gth - 1 | This suffix check is missing a length comparison to correctly handle indexOf returning -1. | | tst.js:67:32:67:71 | x.index ... gth - 1 | This suffix check is missing a length comparison to correctly handle indexOf returning -1. | | tst.js:76:25:76:57 | index = ... gth - 1 | This suffix check is missing a length comparison to correctly handle indexOf returning -1. | +| tst.js:80:10:80:57 | x.index ... th + 1) | This suffix check is missing a length comparison to correctly handle indexOf returning -1. | diff --git a/javascript/ql/test/query-tests/Security/CWE-020/tst.js b/javascript/ql/test/query-tests/Security/CWE-020/tst.js index 6115cf167400..8bbf8cd708f8 100644 --- a/javascript/ql/test/query-tests/Security/CWE-020/tst.js +++ b/javascript/ql/test/query-tests/Security/CWE-020/tst.js @@ -75,3 +75,7 @@ function withIndexOfCheckBad(x, y) { let index = x.indexOf(y); return index !== 0 && index === x.length - y.length - 1; // NOT OK } + +function plus(x, y) { + return x.indexOf("." + y) === x.length - (y.length + 1); // NOT OK +} From 264301be66f15b6915e8d5f7bb6686addac95eb4 Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Thu, 28 Feb 2019 11:36:54 +0100 Subject: [PATCH 40/71] C++: Cache TNode and localFlowStep These two elements weren't cached, which meant that local data flow was recalculated in every query that used data flow. They are also cached in the Java version of `DataFlowUtil.qll`. --- cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll b/cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll index 62dca33d323a..9da1243aa2ef 100644 --- a/cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll +++ b/cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll @@ -4,6 +4,7 @@ import cpp private import semmle.code.cpp.dataflow.internal.FlowVar +cached private newtype TNode = TExprNode(Expr e) or TParameterNode(Parameter p) { exists(p.getFunction().getBlock()) } or @@ -161,6 +162,7 @@ private Variable asVariable(Node node) { * Holds if data flows from `nodeFrom` to `nodeTo` in exactly one local * (intra-procedural) step. */ +cached predicate localFlowStep(Node nodeFrom, Node nodeTo) { // Expr -> Expr exprToExprStep_nocfg(nodeFrom.asExpr(), nodeTo.asExpr()) From 03ef167c56401ace50c24b99c46a7d755a6eb5b9 Mon Sep 17 00:00:00 2001 From: Asger F Date: Thu, 28 Feb 2019 12:37:11 +0000 Subject: [PATCH 41/71] JS: Treat res.end() as alias for res.send() in Express --- javascript/ql/src/semmle/javascript/frameworks/Express.qll | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/frameworks/Express.qll b/javascript/ql/src/semmle/javascript/frameworks/Express.qll index bdddd296a6c4..b6f97f4dde4e 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/Express.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/Express.qll @@ -604,14 +604,15 @@ module Express { } /** - * An argument passed to the `send` method of an HTTP response object. + * An argument passed to the `send` or `end` method of an HTTP response object. */ private class ResponseSendArgument extends HTTP::ResponseSendArgument { RouteHandler rh; ResponseSendArgument() { - exists(MethodCallExpr mce | - mce.calls(rh.getAResponseExpr(), "send") and + exists(MethodCallExpr mce, string name | + mce.calls(rh.getAResponseExpr(), name) and + (name = "send" or name = "end") and this = mce.getArgument(0) ) } From 1444b3976ce9318f0c4058d9ec0b57f593782458 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 28 Feb 2019 11:19:50 +0000 Subject: [PATCH 42/71] Python: Add wsgi.environment as a kind of taint, and add suuport for `env` attribute of falcon request objects. --- .../semmle/python/security/TaintTracking.qll | 33 +++++++++--------- .../python/security/strings/External.qll | 4 +-- python/ql/src/semmle/python/web/Http.qll | 34 +++++++++++++++++++ .../src/semmle/python/web/falcon/Request.qll | 3 +- .../library-tests/web/falcon/Routing.expected | 1 + .../library-tests/web/falcon/Sources.expected | 1 + .../library-tests/web/falcon/Taint.expected | 7 ++++ .../ql/test/library-tests/web/falcon/test.py | 5 +++ 8 files changed, 69 insertions(+), 19 deletions(-) diff --git a/python/ql/src/semmle/python/security/TaintTracking.qll b/python/ql/src/semmle/python/security/TaintTracking.qll index bfedc036fdea..bd0037b52caf 100755 --- a/python/ql/src/semmle/python/security/TaintTracking.qll +++ b/python/ql/src/semmle/python/security/TaintTracking.qll @@ -167,6 +167,7 @@ abstract class CollectionKind extends TaintKind { /* Prevent any collection kinds more than 2 deep */ not this.charAt(2) = "[" and not this.charAt(2) = "{" } + } /** A taint kind representing a flat collections of kinds. @@ -193,7 +194,7 @@ class SequenceKind extends CollectionKind { tonode.(BinaryExprNode).getAnOperand() = fromnode ) or - result = this and copy_call(fromnode, tonode) + result = this and TaintFlowImplementation::copyCall(fromnode, tonode) or exists(BinaryExprNode mod | mod = tonode and @@ -236,20 +237,6 @@ private predicate slice(ControlFlowNode fromnode, SubscriptNode tonode) { ) } -/* A call that returns a copy (or similar) of the argument */ -private predicate copy_call(ControlFlowNode fromnode, CallNode tonode) { - tonode.getFunction().(AttrNode).getObject("copy") = fromnode - or - exists(ModuleObject copy, string name | - name = "copy" or name = "deepcopy" | - copy.attr(name).(FunctionObject).getACall() = tonode and - tonode.getArg(0) = fromnode - ) - or - tonode.getFunction().refersTo(Object::builtin("reversed")) and - tonode.getArg(0) = fromnode -} - /** A taint kind representing a mapping of objects to kinds. * Typically a dict, but can include other mappings. */ @@ -272,7 +259,7 @@ class DictKind extends CollectionKind { result = valueKind and tonode.(CallNode).getFunction().(AttrNode).getObject("get") = fromnode or - result = this and copy_call(fromnode, tonode) + result = this and TaintFlowImplementation::copyCall(fromnode, tonode) or result = this and tonode.(CallNode).getFunction().refersTo(theDictType()) and @@ -1263,6 +1250,20 @@ library module TaintFlowImplementation { context = fromnode.getContext() } + /* A call that returns a copy (or similar) of the argument */ + predicate copyCall(ControlFlowNode fromnode, CallNode tonode) { + tonode.getFunction().(AttrNode).getObject("copy") = fromnode + or + exists(ModuleObject copy, string name | + name = "copy" or name = "deepcopy" | + copy.attr(name).(FunctionObject).getACall() = tonode and + tonode.getArg(0) = fromnode + ) + or + tonode.getFunction().refersTo(Object::builtin("reversed")) and + tonode.getArg(0) = fromnode + } + } /* Helper predicate for tainted_with */ diff --git a/python/ql/src/semmle/python/security/strings/External.qll b/python/ql/src/semmle/python/security/strings/External.qll index cd024b063a74..f9d090ea80ff 100644 --- a/python/ql/src/semmle/python/security/strings/External.qll +++ b/python/ql/src/semmle/python/security/strings/External.qll @@ -96,7 +96,7 @@ private predicate json_load(ControlFlowNode fromnode, CallNode tonode) { ) } -/** A kind of "taint", representing am open file-like object from an external source. */ +/** A kind of "taint", representing an open file-like object from an external source. */ class ExternalFileObject extends TaintKind { ExternalFileObject() { @@ -104,7 +104,7 @@ class ExternalFileObject extends TaintKind { } - /** Gets the taint kind for item in this sequence */ + /** Gets the taint kind for the contents of this file */ TaintKind getValue() { this = "file[" + result + "]" } diff --git a/python/ql/src/semmle/python/web/Http.qll b/python/ql/src/semmle/python/web/Http.qll index 5789fda7d869..0f4d82b3c41a 100644 --- a/python/ql/src/semmle/python/web/Http.qll +++ b/python/ql/src/semmle/python/web/Http.qll @@ -23,3 +23,37 @@ string httpVerb() { string httpVerbLower() { result = httpVerb().toLowerCase() } + +/** Taint kind representing the WSGI environment. + * As specified in PEP 3333. https://www.python.org/dev/peps/pep-3333/#environ-variables + */ +class WsgiEnvironment extends TaintKind { + + WsgiEnvironment() { this = "wsgi.environment" } + + override TaintKind getTaintForFlowStep(ControlFlowNode fromnode, ControlFlowNode tonode) { + result = this and TaintFlowImplementation::copyCall(fromnode, tonode) + or + result = this and + tonode.(CallNode).getFunction().refersTo(theDictType()) and + tonode.(CallNode).getArg(0) = fromnode + or + exists(StringObject key, string text | + tonode.(CallNode).getFunction().(AttrNode).getObject("get") = fromnode and + tonode.(CallNode).getArg(0).refersTo(key) + or + tonode.(SubscriptNode).getValue() = fromnode and tonode.isLoad() and + tonode.(SubscriptNode).getIndex().refersTo(key) + | + text = key.getText() and result instanceof ExternalStringKind and + ( + text = "QUERY_STRING" or + text = "PATH_INFO" or + text.prefix(5) = "HTTP_" + ) + ) + } + +} + + diff --git a/python/ql/src/semmle/python/web/falcon/Request.qll b/python/ql/src/semmle/python/web/falcon/Request.qll index 3cf9563d76f3..94e668402e1d 100644 --- a/python/ql/src/semmle/python/web/falcon/Request.qll +++ b/python/ql/src/semmle/python/web/falcon/Request.qll @@ -13,7 +13,8 @@ class FalconRequest extends TaintKind { } override TaintKind getTaintOfAttribute(string name) { - // name = "env" and result instanceof WsgiEnvironment + name = "env" and result instanceof WsgiEnvironment + or result instanceof ExternalStringKind and ( name = "uri" or name = "url" or diff --git a/python/ql/test/library-tests/web/falcon/Routing.expected b/python/ql/test/library-tests/web/falcon/Routing.expected index 2756730b59c3..47f96f056d85 100644 --- a/python/ql/test/library-tests/web/falcon/Routing.expected +++ b/python/ql/test/library-tests/web/falcon/Routing.expected @@ -1,2 +1,3 @@ +| /hello | delete | test.py:22:5:22:35 | Function on_delete | | /hello | get | test.py:9:5:9:32 | Function on_get | | /hello | post | test.py:19:5:19:33 | Function on_post | diff --git a/python/ql/test/library-tests/web/falcon/Sources.expected b/python/ql/test/library-tests/web/falcon/Sources.expected index 69ea9da4259d..f2ed444a751c 100644 --- a/python/ql/test/library-tests/web/falcon/Sources.expected +++ b/python/ql/test/library-tests/web/falcon/Sources.expected @@ -1,2 +1,3 @@ | test.py:9 | req | falcon.request | | test.py:19 | req | falcon.request | +| test.py:22 | req | falcon.request | \ No newline at end of file diff --git a/python/ql/test/library-tests/web/falcon/Taint.expected b/python/ql/test/library-tests/web/falcon/Taint.expected index 0c7c2d991189..8c00c5886bc3 100644 --- a/python/ql/test/library-tests/web/falcon/Taint.expected +++ b/python/ql/test/library-tests/web/falcon/Taint.expected @@ -16,3 +16,10 @@ | test.py:17 | result | {json[externally controlled string]} | | test.py:19 | req | falcon.request | | test.py:19 | resp | falcon.response | +| test.py:22 | req | falcon.request | +| test.py:22 | resp | falcon.response | +| test.py:23 | Attribute | wsgi.environment | +| test.py:23 | req | falcon.request | +| test.py:24 | Subscript | externally controlled string | +| test.py:24 | env | wsgi.environment | +| test.py:25 | qs | externally controlled string | diff --git a/python/ql/test/library-tests/web/falcon/test.py b/python/ql/test/library-tests/web/falcon/test.py index ff804b172a3e..72853c94ad0a 100644 --- a/python/ql/test/library-tests/web/falcon/test.py +++ b/python/ql/test/library-tests/web/falcon/test.py @@ -19,5 +19,10 @@ def on_get(self, req, resp): def on_post(self, req, resp): pass + def on_delete(self, req, resp): + env = req.env + qs = env["QUERY_STRING"] + return qs + app.add_route('/hello', Handler()) From c4fa29dd0fcc2d1b52a70fe3b9a468a8d68da7b4 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Thu, 28 Feb 2019 14:30:06 +0000 Subject: [PATCH 43/71] JavaScript: Autoformat extractor sources using `google-java-format`. No special settings; command: find javascript/extractor/src -name "*.java" | xargs java -jar /path/to/google-java-format-1.7-all-deps.jar --replace --- .../src/com/semmle/jcorn/CustomParser.java | 1137 +-- .../src/com/semmle/jcorn/ESNextParser.java | 904 ++- .../src/com/semmle/jcorn/Identifiers.java | 251 +- .../src/com/semmle/jcorn/Locutil.java | 37 +- .../src/com/semmle/jcorn/Options.java | 483 +- .../src/com/semmle/jcorn/Parser.java | 7084 +++++++++-------- .../src/com/semmle/jcorn/SyntaxError.java | 18 +- .../src/com/semmle/jcorn/TokenType.java | 470 +- .../src/com/semmle/jcorn/Whitespace.java | 19 +- .../src/com/semmle/jcorn/flow/FlowParser.java | 2474 +++--- .../src/com/semmle/jcorn/jsx/JSXOptions.java | 27 +- .../src/com/semmle/jcorn/jsx/JSXParser.java | 831 +- .../com/semmle/jcorn/jsx/XHTMLEntities.java | 515 +- .../com/semmle/js/ast/ABinaryExpression.java | 39 +- .../src/com/semmle/js/ast/AClass.java | 147 +- .../src/com/semmle/js/ast/AFunction.java | 276 +- .../semmle/js/ast/AFunctionExpression.java | 192 +- .../src/com/semmle/js/ast/AST2JSON.java | 1575 ++-- .../com/semmle/js/ast/ArrayExpression.java | 32 +- .../src/com/semmle/js/ast/ArrayPattern.java | 120 +- .../js/ast/ArrowFunctionExpression.java | 67 +- .../semmle/js/ast/AssignmentExpression.java | 19 +- .../com/semmle/js/ast/AssignmentPattern.java | 23 +- .../com/semmle/js/ast/AwaitExpression.java | 25 +- .../com/semmle/js/ast/BinaryExpression.java | 18 +- .../src/com/semmle/js/ast/BindExpression.java | 38 +- .../src/com/semmle/js/ast/BlockStatement.java | 32 +- .../src/com/semmle/js/ast/BreakStatement.java | 17 +- .../src/com/semmle/js/ast/CallExpression.java | 27 +- .../src/com/semmle/js/ast/CatchClause.java | 70 +- .../src/com/semmle/js/ast/Chainable.java | 16 +- .../src/com/semmle/js/ast/ClassBody.java | 55 +- .../com/semmle/js/ast/ClassDeclaration.java | 88 +- .../com/semmle/js/ast/ClassExpression.java | 74 +- .../src/com/semmle/js/ast/Comment.java | 112 +- .../com/semmle/js/ast/ComprehensionBlock.java | 60 +- .../js/ast/ComprehensionExpression.java | 103 +- .../semmle/js/ast/ConditionalExpression.java | 59 +- .../com/semmle/js/ast/ContinueStatement.java | 17 +- .../com/semmle/js/ast/DebuggerStatement.java | 18 +- .../com/semmle/js/ast/DeclarationFlags.java | 258 +- .../src/com/semmle/js/ast/Decorator.java | 24 +- .../src/com/semmle/js/ast/DefaultVisitor.java | 1406 ++-- .../semmle/js/ast/DestructuringPattern.java | 7 +- .../com/semmle/js/ast/DoWhileStatement.java | 40 +- .../src/com/semmle/js/ast/DynamicImport.java | 24 +- .../src/com/semmle/js/ast/EmptyStatement.java | 18 +- .../semmle/js/ast/EnhancedForStatement.java | 84 +- .../semmle/js/ast/ExportAllDeclaration.java | 24 +- .../com/semmle/js/ast/ExportDeclaration.java | 7 +- .../js/ast/ExportDefaultDeclaration.java | 32 +- .../semmle/js/ast/ExportDefaultSpecifier.java | 19 +- .../semmle/js/ast/ExportNamedDeclaration.java | 77 +- .../js/ast/ExportNamespaceSpecifier.java | 18 +- .../com/semmle/js/ast/ExportSpecifier.java | 42 +- .../src/com/semmle/js/ast/Expression.java | 38 +- .../semmle/js/ast/ExpressionStatement.java | 32 +- .../com/semmle/js/ast/FieldDefinition.java | 103 +- .../src/com/semmle/js/ast/ForInStatement.java | 25 +- .../src/com/semmle/js/ast/ForOfStatement.java | 28 +- .../src/com/semmle/js/ast/ForStatement.java | 103 +- .../semmle/js/ast/FunctionDeclaration.java | 241 +- .../com/semmle/js/ast/FunctionExpression.java | 82 +- .../src/com/semmle/js/ast/IFunction.java | 142 +- .../src/com/semmle/js/ast/INode.java | 16 +- .../src/com/semmle/js/ast/IPattern.java | 7 +- .../src/com/semmle/js/ast/ISourceElement.java | 10 +- .../semmle/js/ast/IStatementContainer.java | 7 +- .../src/com/semmle/js/ast/Identifier.java | 55 +- .../src/com/semmle/js/ast/IfStatement.java | 69 +- .../com/semmle/js/ast/ImportDeclaration.java | 38 +- .../semmle/js/ast/ImportDefaultSpecifier.java | 22 +- .../js/ast/ImportNamespaceSpecifier.java | 20 +- .../com/semmle/js/ast/ImportSpecifier.java | 48 +- .../com/semmle/js/ast/InvokeExpression.java | 133 +- .../src/com/semmle/js/ast/JumpStatement.java | 34 +- .../com/semmle/js/ast/LabeledStatement.java | 46 +- .../src/com/semmle/js/ast/LetExpression.java | 38 +- .../src/com/semmle/js/ast/LetStatement.java | 38 +- .../src/com/semmle/js/ast/Literal.java | 136 +- .../com/semmle/js/ast/LogicalExpression.java | 14 +- .../extractor/src/com/semmle/js/ast/Loop.java | 32 +- .../com/semmle/js/ast/MemberDefinition.java | 233 +- .../com/semmle/js/ast/MemberExpression.java | 113 +- .../src/com/semmle/js/ast/MetaProperty.java | 38 +- .../com/semmle/js/ast/MethodDefinition.java | 107 +- .../src/com/semmle/js/ast/NewExpression.java | 25 +- .../extractor/src/com/semmle/js/ast/Node.java | 20 +- .../src/com/semmle/js/ast/NodeCopier.java | 1498 ++-- .../com/semmle/js/ast/ObjectExpression.java | 32 +- .../src/com/semmle/js/ast/ObjectPattern.java | 102 +- .../js/ast/ParenthesizedExpression.java | 40 +- .../src/com/semmle/js/ast/Position.java | 122 +- .../src/com/semmle/js/ast/Program.java | 71 +- .../src/com/semmle/js/ast/Property.java | 244 +- .../src/com/semmle/js/ast/RestElement.java | 34 +- .../com/semmle/js/ast/ReturnStatement.java | 42 +- .../com/semmle/js/ast/SequenceExpression.java | 32 +- .../src/com/semmle/js/ast/SourceElement.java | 26 +- .../src/com/semmle/js/ast/SourceLocation.java | 147 +- .../src/com/semmle/js/ast/SpreadElement.java | 32 +- .../src/com/semmle/js/ast/Statement.java | 10 +- .../src/com/semmle/js/ast/Super.java | 18 +- .../src/com/semmle/js/ast/SwitchCase.java | 78 +- .../com/semmle/js/ast/SwitchStatement.java | 46 +- .../js/ast/TaggedTemplateExpression.java | 48 +- .../com/semmle/js/ast/TemplateElement.java | 60 +- .../com/semmle/js/ast/TemplateLiteral.java | 113 +- .../src/com/semmle/js/ast/ThisExpression.java | 18 +- .../src/com/semmle/js/ast/ThrowStatement.java | 32 +- .../src/com/semmle/js/ast/Token.java | 107 +- .../src/com/semmle/js/ast/TryStatement.java | 119 +- .../com/semmle/js/ast/UnaryExpression.java | 59 +- .../com/semmle/js/ast/UpdateExpression.java | 62 +- .../semmle/js/ast/VariableDeclaration.java | 92 +- .../com/semmle/js/ast/VariableDeclarator.java | 114 +- .../src/com/semmle/js/ast/Visitor.java | 382 +- .../src/com/semmle/js/ast/WhileStatement.java | 40 +- .../src/com/semmle/js/ast/WithStatement.java | 46 +- .../src/com/semmle/js/ast/XMLAnyName.java | 15 +- .../semmle/js/ast/XMLAttributeSelector.java | 35 +- .../semmle/js/ast/XMLDotDotExpression.java | 33 +- .../semmle/js/ast/XMLFilterExpression.java | 33 +- .../semmle/js/ast/XMLQualifiedIdentifier.java | 44 +- .../com/semmle/js/ast/YieldExpression.java | 46 +- .../com/semmle/js/ast/jsdoc/AllLiteral.java | 26 +- .../com/semmle/js/ast/jsdoc/ArrayType.java | 29 +- .../com/semmle/js/ast/jsdoc/CompoundType.java | 44 +- .../com/semmle/js/ast/jsdoc/FieldType.java | 80 +- .../com/semmle/js/ast/jsdoc/FunctionType.java | 183 +- .../com/semmle/js/ast/jsdoc/JSDocComment.java | 65 +- .../com/semmle/js/ast/jsdoc/JSDocElement.java | 16 +- .../src/com/semmle/js/ast/jsdoc/JSDocTag.java | 127 +- .../js/ast/jsdoc/JSDocTypeExpression.java | 30 +- .../semmle/js/ast/jsdoc/NameExpression.java | 40 +- .../semmle/js/ast/jsdoc/NonNullableType.java | 18 +- .../com/semmle/js/ast/jsdoc/NullLiteral.java | 26 +- .../semmle/js/ast/jsdoc/NullableLiteral.java | 26 +- .../com/semmle/js/ast/jsdoc/NullableType.java | 18 +- .../com/semmle/js/ast/jsdoc/OptionalType.java | 18 +- .../semmle/js/ast/jsdoc/ParameterType.java | 54 +- .../com/semmle/js/ast/jsdoc/RecordType.java | 59 +- .../src/com/semmle/js/ast/jsdoc/RestType.java | 18 +- .../semmle/js/ast/jsdoc/TypeApplication.java | 94 +- .../js/ast/jsdoc/UnaryTypeConstructor.java | 58 +- .../semmle/js/ast/jsdoc/UndefinedLiteral.java | 26 +- .../com/semmle/js/ast/jsdoc/UnionType.java | 29 +- .../src/com/semmle/js/ast/jsdoc/Visitor.java | 60 +- .../com/semmle/js/ast/jsdoc/VoidLiteral.java | 26 +- .../com/semmle/js/ast/jsdoc/package-info.java | 14 +- .../src/com/semmle/js/ast/json/JSONArray.java | 59 +- .../com/semmle/js/ast/json/JSONLiteral.java | 76 +- .../com/semmle/js/ast/json/JSONObject.java | 63 +- .../src/com/semmle/js/ast/json/JSONValue.java | 30 +- .../src/com/semmle/js/ast/json/Visitor.java | 10 +- .../com/semmle/js/ast/json/package-info.java | 12 +- .../com/semmle/js/ast/jsx/IJSXAttribute.java | 3 +- .../com/semmle/js/ast/jsx/IJSXExpression.java | 4 +- .../src/com/semmle/js/ast/jsx/IJSXName.java | 2 +- .../com/semmle/js/ast/jsx/JSXAttribute.java | 34 +- .../semmle/js/ast/jsx/JSXBoundaryElement.java | 2 +- .../semmle/js/ast/jsx/JSXClosingElement.java | 27 +- .../src/com/semmle/js/ast/jsx/JSXElement.java | 61 +- .../semmle/js/ast/jsx/JSXEmptyExpression.java | 14 +- .../js/ast/jsx/JSXExpressionContainer.java | 24 +- .../com/semmle/js/ast/jsx/JSXIdentifier.java | 22 +- .../js/ast/jsx/JSXMemberExpression.java | 42 +- .../semmle/js/ast/jsx/JSXNamespacedName.java | 40 +- .../semmle/js/ast/jsx/JSXOpeningElement.java | 60 +- .../semmle/js/ast/jsx/JSXSpreadAttribute.java | 24 +- .../src/com/semmle/js/ast/package-info.java | 37 +- .../semmle/js/ast/regexp/BackReference.java | 46 +- .../src/com/semmle/js/ast/regexp/Caret.java | 18 +- .../semmle/js/ast/regexp/CharacterClass.java | 49 +- .../js/ast/regexp/CharacterClassEscape.java | 44 +- .../js/ast/regexp/CharacterClassRange.java | 44 +- .../com/semmle/js/ast/regexp/Constant.java | 18 +- .../semmle/js/ast/regexp/ControlEscape.java | 21 +- .../semmle/js/ast/regexp/ControlLetter.java | 21 +- .../semmle/js/ast/regexp/DecimalEscape.java | 18 +- .../com/semmle/js/ast/regexp/Disjunction.java | 35 +- .../src/com/semmle/js/ast/regexp/Dollar.java | 18 +- .../src/com/semmle/js/ast/regexp/Dot.java | 18 +- .../src/com/semmle/js/ast/regexp/Error.java | 50 +- .../semmle/js/ast/regexp/EscapeSequence.java | 39 +- .../src/com/semmle/js/ast/regexp/Group.java | 99 +- .../js/ast/regexp/HexEscapeSequence.java | 18 +- .../semmle/js/ast/regexp/IdentityEscape.java | 21 +- .../src/com/semmle/js/ast/regexp/Literal.java | 24 +- .../js/ast/regexp/NamedBackReference.java | 46 +- .../semmle/js/ast/regexp/NonWordBoundary.java | 18 +- .../com/semmle/js/ast/regexp/OctalEscape.java | 18 +- .../src/com/semmle/js/ast/regexp/Opt.java | 18 +- .../src/com/semmle/js/ast/regexp/Plus.java | 18 +- .../com/semmle/js/ast/regexp/Quantifier.java | 38 +- .../src/com/semmle/js/ast/regexp/Range.java | 54 +- .../com/semmle/js/ast/regexp/RegExpTerm.java | 30 +- .../com/semmle/js/ast/regexp/Sequence.java | 35 +- .../src/com/semmle/js/ast/regexp/Star.java | 18 +- .../js/ast/regexp/UnicodeEscapeSequence.java | 18 +- .../js/ast/regexp/UnicodePropertyEscape.java | 78 +- .../src/com/semmle/js/ast/regexp/Visitor.java | 93 +- .../semmle/js/ast/regexp/WordBoundary.java | 18 +- .../regexp/ZeroWidthNegativeLookahead.java | 34 +- .../regexp/ZeroWidthNegativeLookbehind.java | 34 +- .../regexp/ZeroWidthPositiveLookahead.java | 34 +- .../regexp/ZeroWidthPositiveLookbehind.java | 34 +- .../semmle/js/ast/regexp/package-info.java | 11 +- .../com/semmle/js/extractor/ASTExtractor.java | 3587 ++++----- .../com/semmle/js/extractor/AutoBuild.java | 1390 ++-- .../com/semmle/js/extractor/CFGExtractor.java | 3740 +++++---- .../semmle/js/extractor/DeclaredNames.java | 67 +- .../com/semmle/js/extractor/ExprKinds.java | 541 +- .../semmle/js/extractor/ExtractorConfig.java | 839 +- .../semmle/js/extractor/ExtractorState.java | 43 +- .../semmle/js/extractor/FileExtractor.java | 968 ++- .../semmle/js/extractor/HTMLExtractor.java | 642 +- .../com/semmle/js/extractor/IExtractor.java | 16 +- .../semmle/js/extractor/JSDocExtractor.java | 437 +- .../com/semmle/js/extractor/JSExtractor.java | 256 +- .../semmle/js/extractor/JSONExtractor.java | 168 +- .../src/com/semmle/js/extractor/JumpType.java | 11 +- .../semmle/js/extractor/LexicalExtractor.java | 366 +- .../src/com/semmle/js/extractor/LoCInfo.java | 34 +- .../semmle/js/extractor/LocationManager.java | 227 +- .../src/com/semmle/js/extractor/Main.java | 915 +-- .../semmle/js/extractor/NodeJSDetector.java | 311 +- .../semmle/js/extractor/RegExpExtractor.java | 635 +- .../com/semmle/js/extractor/ScopeManager.java | 1318 ++- .../semmle/js/extractor/ScriptExtractor.java | 109 +- .../com/semmle/js/extractor/StmtKinds.java | 143 +- .../js/extractor/SyntacticContextManager.java | 249 +- .../semmle/js/extractor/TextualExtractor.java | 255 +- .../semmle/js/extractor/TypeExprKinds.java | 416 +- .../js/extractor/TypeScriptExtractor.java | 46 +- .../semmle/js/extractor/TypeScriptMode.java | 30 +- .../semmle/js/extractor/YAMLExtractor.java | 426 +- .../js/extractor/test/ASTMatchingTests.java | 136 +- .../semmle/js/extractor/test/AllTests.java | 26 +- .../js/extractor/test/AutoBuildTests.java | 955 ++- .../extractor/test/ClassPropertiesTests.java | 74 +- .../js/extractor/test/DecoratorTests.java | 102 +- .../extractor/test/ExportExtensionsTests.java | 67 +- .../js/extractor/test/FunctionSentTests.java | 30 +- .../semmle/js/extractor/test/JSXTests.java | 134 +- .../extractor/test/NodeJSDetectorTests.java | 380 +- .../extractor/test/NumericSeparatorTests.java | 58 +- .../extractor/test/ObjectRestSpreadTests.java | 95 +- .../js/extractor/test/RobustnessTests.java | 21 +- .../semmle/js/extractor/test/TrapTests.java | 321 +- .../trapcache/CachingTrapWriter.java | 114 +- .../extractor/trapcache/DefaultTrapCache.java | 273 +- .../extractor/trapcache/DummyTrapCache.java | 15 +- .../js/extractor/trapcache/ITrapCache.java | 30 +- .../src/com/semmle/js/parser/JSDocParser.java | 3624 +++++---- .../src/com/semmle/js/parser/JSONParser.java | 745 +- .../src/com/semmle/js/parser/JSParser.java | 108 +- .../com/semmle/js/parser/JcornWrapper.java | 77 +- .../src/com/semmle/js/parser/ParseError.java | 87 +- .../com/semmle/js/parser/ParsedProject.java | 34 +- .../com/semmle/js/parser/RegExpParser.java | 915 +-- .../js/parser/TypeScriptASTConverter.java | 4617 +++++------ .../semmle/js/parser/TypeScriptParser.java | 804 +- .../src/com/semmle/ts/ast/ArrayTypeExpr.java | 27 +- .../semmle/ts/ast/ConditionalTypeExpr.java | 76 +- .../src/com/semmle/ts/ast/DecoratorList.java | 27 +- .../com/semmle/ts/ast/EnumDeclaration.java | 110 +- .../src/com/semmle/ts/ast/EnumMember.java | 72 +- .../ts/ast/ExportAsNamespaceDeclaration.java | 29 +- .../semmle/ts/ast/ExportWholeDeclaration.java | 24 +- .../ts/ast/ExpressionWithTypeArguments.java | 45 +- .../ts/ast/ExternalModuleDeclaration.java | 41 +- .../ts/ast/ExternalModuleReference.java | 25 +- .../com/semmle/ts/ast/FunctionTypeExpr.java | 34 +- .../com/semmle/ts/ast/GenericTypeExpr.java | 42 +- .../ts/ast/GlobalAugmentationDeclaration.java | 31 +- .../com/semmle/ts/ast/INodeWithSymbol.java | 16 +- .../com/semmle/ts/ast/ITypeExpression.java | 12 +- .../src/com/semmle/ts/ast/ITypedAstNode.java | 22 +- .../src/com/semmle/ts/ast/ImportTypeExpr.java | 28 +- .../semmle/ts/ast/ImportWholeDeclaration.java | 38 +- .../semmle/ts/ast/IndexedAccessTypeExpr.java | 39 +- .../src/com/semmle/ts/ast/InferTypeExpr.java | 28 +- .../semmle/ts/ast/InterfaceDeclaration.java | 92 +- .../com/semmle/ts/ast/InterfaceTypeExpr.java | 31 +- .../semmle/ts/ast/IntersectionTypeExpr.java | 33 +- .../src/com/semmle/ts/ast/IsTypeExpr.java | 38 +- .../src/com/semmle/ts/ast/KeyofTypeExpr.java | 28 +- .../com/semmle/ts/ast/KeywordTypeExpr.java | 38 +- .../src/com/semmle/ts/ast/MappedTypeExpr.java | 44 +- .../semmle/ts/ast/NamespaceDeclaration.java | 110 +- .../com/semmle/ts/ast/NonNullAssertion.java | 28 +- .../com/semmle/ts/ast/OptionalTypeExpr.java | 29 +- .../semmle/ts/ast/ParenthesizedTypeExpr.java | 28 +- .../src/com/semmle/ts/ast/RestTypeExpr.java | 29 +- .../src/com/semmle/ts/ast/TupleTypeExpr.java | 31 +- .../semmle/ts/ast/TypeAliasDeclaration.java | 91 +- .../src/com/semmle/ts/ast/TypeAssertion.java | 60 +- .../src/com/semmle/ts/ast/TypeExpression.java | 27 +- .../src/com/semmle/ts/ast/TypeParameter.java | 72 +- .../src/com/semmle/ts/ast/TypeofTypeExpr.java | 28 +- .../src/com/semmle/ts/ast/UnionTypeExpr.java | 29 +- .../semmle/ts/extractor/TypeExtractor.java | 499 +- .../com/semmle/ts/extractor/TypeTable.java | 206 +- 304 files changed, 31625 insertions(+), 31631 deletions(-) diff --git a/javascript/extractor/src/com/semmle/jcorn/CustomParser.java b/javascript/extractor/src/com/semmle/jcorn/CustomParser.java index 188a6183af79..a77548ed21f5 100644 --- a/javascript/extractor/src/com/semmle/jcorn/CustomParser.java +++ b/javascript/extractor/src/com/semmle/jcorn/CustomParser.java @@ -1,9 +1,5 @@ package com.semmle.jcorn; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; - import com.semmle.jcorn.TokenType.Properties; import com.semmle.jcorn.flow.FlowParser; import com.semmle.js.ast.ArrayExpression; @@ -41,557 +37,592 @@ import com.semmle.js.ast.XMLFilterExpression; import com.semmle.util.data.Either; import com.semmle.util.data.Pair; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; /** - * An extension of the standard jcorn parser with support for Mozilla-specific - * language extension (most of JavaScript 1.8.5 and E4X) and JScript language extensions. + * An extension of the standard jcorn parser with support for Mozilla-specific language extension + * (most of JavaScript 1.8.5 and E4X) and JScript language extensions. */ public class CustomParser extends FlowParser { - public CustomParser(Options options, String input, int startPos) { - super(options, input, startPos); - - // recognise `const` as a keyword, irrespective of options.ecmaVersion - this.keywords.add("const"); - } - - // add parsing of guarded `catch` clauses - @Override - protected TryStatement parseTryStatement(Position startLoc) { - if (!options.mozExtensions()) - return super.parseTryStatement(startLoc); - - this.next(); - BlockStatement block = this.parseBlock(false); - CatchClause handler = null; - List guardedHandlers = new ArrayList(); - while (this.type == TokenType._catch) { - Position catchStartLoc = this.startLoc; - CatchClause katch = this.parseCatchClause(catchStartLoc); - if (handler != null) - this.raise(catchStartLoc, "Catch after unconditional catch"); - if (katch.getGuard() != null) - guardedHandlers.add(katch); - else - handler = katch; - } - BlockStatement finalizer = this.eat(TokenType._finally) ? this.parseBlock(false) : null; - if (handler == null && finalizer == null && guardedHandlers.isEmpty()) - this.raise(startLoc, "Missing catch or finally clause"); - return this.finishNode(new TryStatement(new SourceLocation(startLoc), block, handler, guardedHandlers, finalizer)); - } - - /* - * Support for guarded `catch` clauses and omitted catch bindings. - */ - @Override - protected CatchClause parseCatchClause(Position startLoc) { - if (!options.mozExtensions()) - return super.parseCatchClause(startLoc); - - this.next(); - Expression param = null; - Expression guard = null; - if (this.eat(TokenType.parenL)) { - param = this.parseBindingAtom(); - this.checkLVal(param, true, null); - if (this.eat(TokenType._if)) - guard = this.parseExpression(false, null); - this.expect(TokenType.parenR); - } else if (!options.esnext()) { - this.unexpected(); - } - BlockStatement catchBody = this.parseBlock(false); - return this.finishNode(new CatchClause(new SourceLocation(startLoc), (IPattern)param, guard, catchBody)); - } - - // add parsing of `let` statements and expressions - @Override - protected boolean mayFollowLet(int c) { - return options.mozExtensions() && c == '(' || super.mayFollowLet(c); - } - - @Override - protected Statement parseVarStatement(Position startLoc, String kind) { - if (!options.mozExtensions()) - return super.parseVarStatement(startLoc, kind); - - this.next(); - - if ("let".equals(kind) && this.eat(TokenType.parenL)) { - // this is a `let` statement or expression - return (LetStatement) this.parseLetExpression(startLoc, true); - } - - VariableDeclaration node = this.parseVar(startLoc, false, kind); - this.semicolon(); - return this.finishNode(node); - } - - @Override - protected Expression parseExprAtom(DestructuringErrors refDestructuringErrors) { - Position startLoc = this.startLoc; - if (options.mozExtensions() && this.isContextual("let")) { - this.next(); - this.expect(TokenType.parenL); - return (Expression) this.parseLetExpression(startLoc, false); - } else if (options.mozExtensions() && this.type == TokenType.bracketL) { - this.next(); - // check whether this is array comprehension or regular array - if (this.type == TokenType._for) { - ComprehensionExpression c = this.parseComprehension(startLoc, false, null); - this.expect(TokenType.bracketR); - return this.finishNode(c); - } - List elements; - if (this.type == TokenType.comma || this.type == TokenType.bracketR || - this.type == TokenType.ellipsis) { - elements = this.parseExprList(TokenType.bracketR, true, true, refDestructuringErrors); - } else { - Expression firstExpr = this.parseMaybeAssign(false, refDestructuringErrors, null); - // check whether this is a postfix array comprehension - if (this.type == TokenType._for || this.type == TokenType._if) { - ComprehensionExpression c = this.parseComprehension(startLoc, false, firstExpr); - this.expect(TokenType.bracketR); - return this.finishNode(c); - } else { - this.eat(TokenType.comma); - elements = new ArrayList(); - elements.add(firstExpr); - elements.addAll(this.parseExprList(TokenType.bracketR, true, true, refDestructuringErrors)); - } - } - return this.finishNode(new ArrayExpression(new SourceLocation(startLoc), elements)); - } else if (options.v8Extensions() && this.type == TokenType.modulo) { - // parse V8 native - this.next(); - Identifier name = this.parseIdent(true); - this.expect(TokenType.parenL); - List args = this.parseExprList(TokenType.parenR, false, false, null); - CallExpression node = new CallExpression(new SourceLocation(startLoc), name, new ArrayList<>(), args, false, false); - return this.finishNode(node); - } else if (options.e4x() && this.type == at) { - // this could be either a decorator or an attribute selector; we first - // try parsing it as a decorator, and then convert it to an attribute selector - // if the next token turns out not to be `class` - List decorators = parseDecorators(); - Expression attr = null; - if (decorators.size() > 1 || - this.type == TokenType._class || - ((attr = decoratorToAttributeSelector(decorators.get(0))) == null)) { - ClassExpression ce = (ClassExpression) this.parseClass(startLoc, false); - ce.addDecorators(decorators); - return ce; - } - return attr; - } else { - return super.parseExprAtom(refDestructuringErrors); - } - } - - protected Node parseLetExpression(Position startLoc, boolean maybeStatement) { - // this method assumes that the keyword `let` and the opening parenthesis have already been - // consumed - VariableDeclaration decl = this.parseVar(startLoc, false, "let"); - this.expect(TokenType.parenR); - - if (this.type == TokenType.braceL) { - if (!maybeStatement) { - // must be the start of an object literal - Expression body = this.parseObj(false, null); - return this.finishNode(new LetExpression(new SourceLocation(startLoc), decl.getDeclarations(), body)); - } - - BlockStatement body = this.parseBlock(false); - return this.finishNode(new LetStatement(new SourceLocation(startLoc), decl.getDeclarations(), body)); - } else if (maybeStatement) { - Position pos = startLoc; - Statement body = this.parseStatement(true, false); - if (body == null) - this.unexpected(pos); - return this.finishNode(new LetStatement(new SourceLocation(startLoc), decl.getDeclarations(), body)); - } else { - Expression body = this.parseExpression(false, null); - return this.finishNode(new LetExpression(new SourceLocation(startLoc), decl.getDeclarations(), body)); - } - } - - // add parsing of expression closures and JScript methods - @Override - protected INode parseFunction(Position startLoc, boolean isStatement, boolean allowExpressionBody, boolean isAsync) { - if (isFunctionSent(isStatement)) - return super.parseFunction(startLoc, isStatement, allowExpressionBody, isAsync); - allowExpressionBody = allowExpressionBody || options.mozExtensions(); - boolean oldInGen = this.inGenerator, oldInAsync = this.inAsync; - int oldYieldPos = this.yieldPos, oldAwaitPos = this.awaitPos; - Pair p = parseFunctionName(isStatement, isAsync); - boolean generator = p.fst(); - Identifier id = p.snd(), iface = null; - if (options.jscript()) { - if (isStatement && this.eatDoubleColon()) { - iface = p.snd(); - id = this.parseIdent(false); - } - } - IFunction result = parseFunctionRest(startLoc, isStatement, allowExpressionBody, oldInGen, oldInAsync, - oldYieldPos, oldAwaitPos, generator, id); - if (iface != null) { - /* Translate JScript double colon method declarations into normal method definitions: - * - * function A::f(...) { ... } - * - * becomes - * - * A.f = function f(...) { ... }; - */ - SourceLocation memloc = new SourceLocation(iface.getName() + "::" + id.getName(), iface.getLoc().getStart(), id.getLoc().getEnd()); - MemberExpression mem = new MemberExpression(memloc, iface, new Identifier(id.getLoc(), id.getName()), false, false, false); - AssignmentExpression assgn = new AssignmentExpression(result.getLoc(), "=", mem, ((FunctionDeclaration)result).asFunctionExpression()); - return new ExpressionStatement(result.getLoc(), assgn); - } - return result; - } - - private boolean eatDoubleColon() { - if (this.eat(TokenType.colon)) { - this.expect(TokenType.colon); - return true; - } else { - return this.eat(doubleColon); - } - } - - // accept `yield` in non-generator functions - @Override - protected Expression parseMaybeAssign(boolean noIn, - DestructuringErrors refDestructuringErrors, - AfterLeftParse afterLeftParse) { - if (options.mozExtensions() && isContextual("yield")) { - if (!this.inFunction) - this.raise(this.startLoc, "Yield not in function"); - return this.parseYield(); - } - return super.parseMaybeAssign(noIn, refDestructuringErrors, afterLeftParse); - } - - // add parsing of comprehensions - protected ComprehensionExpression parseComprehension(Position startLoc, boolean isGenerator, Expression body) { - List blocks = new ArrayList(); - while (this.type == TokenType._for) { - SourceLocation blockStart = new SourceLocation(this.startLoc); - boolean of = false; - this.next(); - if (this.eatContextual("each")) - of = true; - this.expect(TokenType.parenL); - Expression left = this.parseBindingAtom(); - this.checkLVal(left, true, null); - if (this.eatContextual("of")) { - of = true; - } else { - this.expect(TokenType._in); - } - Expression right = this.parseExpression(false, null); - this.expect(TokenType.parenR); - blocks.add(this.finishNode(new ComprehensionBlock(blockStart, (IPattern)left, right, of))); - } - Expression filter = this.eat(TokenType._if) ? this.parseParenExpression() : null; - if (body == null) - body = this.parseExpression(false, null); - - return new ComprehensionExpression(new SourceLocation(startLoc), body, blocks, filter, isGenerator); - } - - @Override - protected Expression parseParenAndDistinguishExpression(boolean canBeArrow) { - if (options.mozExtensions()) { - // check whether next token is `for`, suggesting a generator comprehension - Position startLoc = this.startLoc; - Matcher m = Whitespace.skipWhiteSpace.matcher(this.input); - if (m.find(this.pos)) { - if (m.end()+3 < input.length() && - "for".equals(input.substring(m.end(), m.end()+3)) && - !Identifiers.isIdentifierChar(input.charAt(m.end()+3), true)) { - next(); - ComprehensionExpression c = parseComprehension(startLoc, true, null); - this.expect(TokenType.parenR); - return this.finishNode(c); - } - } - } - - Expression res = super.parseParenAndDistinguishExpression(canBeArrow); - if (res instanceof ParenthesizedExpression) { - ParenthesizedExpression p = (ParenthesizedExpression) res; - if (p.getExpression() instanceof ComprehensionExpression) { - ComprehensionExpression c = (ComprehensionExpression) p.getExpression(); - if (c.isGenerator()) - return new ComprehensionExpression(p.getLoc(), c.getBody(), c.getBlocks(), c.getFilter(), c.isGenerator()); - } - } - return res; - } - - @Override - protected boolean parseParenthesisedExpression(DestructuringErrors refDestructuringErrors, - boolean allowTrailingComma, ParenthesisedExpressions parenExprs, boolean first) { - boolean cont = super.parseParenthesisedExpression(refDestructuringErrors, allowTrailingComma, parenExprs, first); - if (options.mozExtensions() && parenExprs.exprList.size() == 1 && this.type == TokenType._for) { - Expression body = parenExprs.exprList.remove(0); - ComprehensionExpression c = parseComprehension(body.getLoc().getStart(), true, body); - parenExprs.exprList.add(this.finishNode(c)); - return false; - } - return cont; - } - - // add parsing of for-each loops - @Override - protected Statement parseForStatement(Position startLoc) { - boolean each = false; - if (options.mozExtensions() && this.isContextual("each")) { - this.next(); - each = true; - } - Position afterEach = this.startLoc; - Statement result = super.parseForStatement(startLoc); - if (each) { - if (result instanceof ForInStatement) { - ForInStatement fis = (ForInStatement) result; - result = new ForInStatement(fis.getLoc(), fis.getLeft(), fis.getRight(), fis.getBody(), true); - } else { - raise(afterEach, "Bad for-each statement."); - } - } - return result; - } - - // add parsing of Rhino/Nashorn-style `new` expressions with last argument after `)` - @Override - protected Expression parseNew() { - Expression res = super.parseNew(); - if (res instanceof NewExpression && - options.mozExtensions() && !canInsertSemicolon() && this.type == TokenType.braceL) { - ((NewExpression) res).getArguments().add(this.parseObj(false, null)); - res = this.finishNode(res); - } - return res; - } - - /* - * E4X - * - * PrimaryExpression : - * PropertyIdentifier - * XMLInitialiser - * XMLListInitialiser - * - * PropertyIdentifier : - * AttributeIdentifier - * QualifiedIdentifier - * WildcardIdent - * - * AttributeIdentifier : - * @ PropertySelector - * @ QualifiedIdentifier - * @ [ Expression ] - * - * PropertySelector : - * Identifier - * WildcardIdentifier - * - * QualifiedIdentifier : - * PropertySelector :: PropertySelector - * PropertySelector :: [ Expression ] - * - * WildcardIdentifier : - * * - * - * MemberExpression : - * MemberExpression . PropertyIdentifier - * MemberExpression .. Identifier - * MemberExpression .. PropertyIdentifier - * MemberExpression . ( Expression ) - * - * DefaultXMLNamespaceStatement : - * default xml namespace = Expression - */ - - protected TokenType doubleDot = new TokenType(new Properties(":").beforeExpr()); - - @Override - protected Token getTokenFromCode(int code) { - if (options.e4x() && code == '.' && charAt(this.pos+1) == '.' && charAt(this.pos+2) != '.') { - this.pos += 2; - return this.finishToken(doubleDot); - } - return super.getTokenFromCode(code); - } - - // add parsing of E4X property, attribute and descendant accesses, as well as filter expressions - @Override - protected Pair parseSubscript(Expression base, Position startLoc, boolean noCalls) { - if (options.e4x() && this.eat(TokenType.dot)) { - SourceLocation start = new SourceLocation(startLoc); - if (this.eat(TokenType.parenL)) { - Expression filter = parseExpression(false, null); - this.expect(TokenType.parenR); - return Pair.make(this.finishNode(new XMLFilterExpression(start, base, filter)), true); - } - - Expression property = this.parsePropertyIdentifierOrIdentifier(); - MemberExpression node = new MemberExpression(start, base, property, false, false, isOnOptionalChain(false, base)); - return Pair.make(this.finishNode(node), true); - } else if (this.eat(doubleDot)) { - SourceLocation start = new SourceLocation(startLoc); - Expression property = this.parsePropertyIdentifierOrIdentifier(); - return Pair.make(this.finishNode(new XMLDotDotExpression(start, base, property)), true); - } - return super.parseSubscript(base, startLoc, noCalls); - } - - /** - * Parse a an attribute identifier, a wildcard identifier, a qualified identifier, - * or a plain identifier. - */ - protected Expression parsePropertyIdentifierOrIdentifier() { - Position start = this.startLoc; - if (this.eat(at)) { - // attribute identifier - return parseAttributeIdentifier(new SourceLocation(start)); - } else { - return parsePropertySelector(new SourceLocation(startLoc)); - } - } - - /** - * Parse a property selector, that is, either a wildcard identifier or a plain identifier. - */ - protected Expression parsePropertySelector(SourceLocation start) { - Expression res; - if (this.eat(TokenType.star)) { - // wildcard identifier - res = this.finishNode(new XMLAnyName(start)); - } else { - res = this.parseIdent(true); - } - return res; - } - - /** - * Parse an attribute identifier, either computed ({@code [ Expr ]}) or a possibly - * qualified identifier. - */ - protected Expression parseAttributeIdentifier(SourceLocation start) { - if (this.eat(TokenType.bracketL)) { - Expression idx = parseExpression(false, null); - this.expect(TokenType.bracketR); - return this.finishNode(new XMLAttributeSelector(start, idx, true)); - } else { - return this.finishNode(new XMLAttributeSelector(start, parsePropertySelector(new SourceLocation(startLoc)), false)); - } - } - - @Override - protected Expression parseDecoratorBody() { - SourceLocation start = new SourceLocation(startLoc); - if (options.e4x() && this.eat(TokenType.bracketL)) { - // this must be an attribute selector, so only allow a single expression - // followed by a right bracket, which will later be converted by - // `decoratorToAttributeSelector` below - List elements = new ArrayList<>(); - elements.add(parseExpression(false, null)); - this.expect(TokenType.bracketR); - return this.finishNode(new ArrayExpression(start, elements)); - } - - return super.parseDecoratorBody(); - } - - /** - * Convert a decorator that resulted from mis-parsing an attribute selector into - * an attribute selector. - */ - protected XMLAttributeSelector decoratorToAttributeSelector(Decorator d) { - Expression e = d.getExpression(); - if (e instanceof ArrayExpression) { - ArrayExpression ae = (ArrayExpression) e; - if (ae.getElements().size() == 1) - return new XMLAttributeSelector(d.getLoc(), ae.getElements().get(0), true); - } else if (e instanceof Identifier) { - return new XMLAttributeSelector(d.getLoc(), e, false); - } - return null; - } - - @Override - protected Token readToken(int code) { - // skip XML processing instructions (which are allowed in E4X, but not in JSX); - // there is a lexical ambiguity between an XML processing instruction starting a - // chunk of E4X content and a Flow type annotation (both can start with `` of a putative XML processing instruction - // we backtrack and try lexing as something else - // to avoid frequent backtracking, we only consider `` processing instructions; - // while other processing instructions are technically possible, they are unlikely in practice - if (this.options.e4x()) { - while (code == '<') { - if (inputSubstring(this.pos+1, this.pos+5).equals("?xml")) { - int oldPos = this.pos; - this.pos += 5; - if (!jsx_readUntil("?>")) { - // didn't find a closing `?>`, so backtrack - this.pos = oldPos; - break; - } - } else { - break; - } - this.skipSpace(); - code = this.fullCharCodeAtPos(); - } - } - return super.readToken(code); - } - - @Override - protected Either jsx_readChunk(StringBuilder out, int chunkStart, int ch) { - // skip XML comments, processing instructions and CDATA (which are allowed in E4X, - // but not in JSX) - // unlike in `readToken` above, we know that we're inside JSX/E4X code, so there is - // no ambiguity with Flow type annotations - if (this.options.e4x() && ch == '<') { - if (inputSubstring(this.pos+1, this.pos+4).equals("!--")) { - out.append(inputSubstring(chunkStart, this.pos)); - this.pos += 4; - jsx_readUntil("-->"); - return Either.left(this.pos); - } else if (charAt(this.pos+1) == '?') { - out.append(inputSubstring(chunkStart, this.pos)); - this.pos += 2; - jsx_readUntil("?>"); - return Either.left(this.pos); - } else if (inputSubstring(this.pos+1, this.pos+9).equals("![CDATA[")) { - out.append(inputSubstring(chunkStart, this.pos)); - this.pos += 9; - int cdataStart = this.pos; - jsx_readUntil("]]>"); - out.append(inputSubstring(cdataStart, this.pos-3)); - return Either.left(this.pos); - } - } - - return super.jsx_readChunk(out, chunkStart, ch); - } - - private boolean jsx_readUntil(String terminator) { - char fst = terminator.charAt(0); - while (this.pos+terminator.length() <= this.input.length()) { - if (charAt(this.pos) == fst && - inputSubstring(this.pos, this.pos+terminator.length()).equals(terminator)) { - this.pos += terminator.length(); - return true; - } - ++this.pos; - } - return false; - } + public CustomParser(Options options, String input, int startPos) { + super(options, input, startPos); + + // recognise `const` as a keyword, irrespective of options.ecmaVersion + this.keywords.add("const"); + } + + // add parsing of guarded `catch` clauses + @Override + protected TryStatement parseTryStatement(Position startLoc) { + if (!options.mozExtensions()) return super.parseTryStatement(startLoc); + + this.next(); + BlockStatement block = this.parseBlock(false); + CatchClause handler = null; + List guardedHandlers = new ArrayList(); + while (this.type == TokenType._catch) { + Position catchStartLoc = this.startLoc; + CatchClause katch = this.parseCatchClause(catchStartLoc); + if (handler != null) this.raise(catchStartLoc, "Catch after unconditional catch"); + if (katch.getGuard() != null) guardedHandlers.add(katch); + else handler = katch; + } + BlockStatement finalizer = this.eat(TokenType._finally) ? this.parseBlock(false) : null; + if (handler == null && finalizer == null && guardedHandlers.isEmpty()) + this.raise(startLoc, "Missing catch or finally clause"); + return this.finishNode( + new TryStatement(new SourceLocation(startLoc), block, handler, guardedHandlers, finalizer)); + } + + /* + * Support for guarded `catch` clauses and omitted catch bindings. + */ + @Override + protected CatchClause parseCatchClause(Position startLoc) { + if (!options.mozExtensions()) return super.parseCatchClause(startLoc); + + this.next(); + Expression param = null; + Expression guard = null; + if (this.eat(TokenType.parenL)) { + param = this.parseBindingAtom(); + this.checkLVal(param, true, null); + if (this.eat(TokenType._if)) guard = this.parseExpression(false, null); + this.expect(TokenType.parenR); + } else if (!options.esnext()) { + this.unexpected(); + } + BlockStatement catchBody = this.parseBlock(false); + return this.finishNode( + new CatchClause(new SourceLocation(startLoc), (IPattern) param, guard, catchBody)); + } + + // add parsing of `let` statements and expressions + @Override + protected boolean mayFollowLet(int c) { + return options.mozExtensions() && c == '(' || super.mayFollowLet(c); + } + + @Override + protected Statement parseVarStatement(Position startLoc, String kind) { + if (!options.mozExtensions()) return super.parseVarStatement(startLoc, kind); + + this.next(); + + if ("let".equals(kind) && this.eat(TokenType.parenL)) { + // this is a `let` statement or expression + return (LetStatement) this.parseLetExpression(startLoc, true); + } + + VariableDeclaration node = this.parseVar(startLoc, false, kind); + this.semicolon(); + return this.finishNode(node); + } + + @Override + protected Expression parseExprAtom(DestructuringErrors refDestructuringErrors) { + Position startLoc = this.startLoc; + if (options.mozExtensions() && this.isContextual("let")) { + this.next(); + this.expect(TokenType.parenL); + return (Expression) this.parseLetExpression(startLoc, false); + } else if (options.mozExtensions() && this.type == TokenType.bracketL) { + this.next(); + // check whether this is array comprehension or regular array + if (this.type == TokenType._for) { + ComprehensionExpression c = this.parseComprehension(startLoc, false, null); + this.expect(TokenType.bracketR); + return this.finishNode(c); + } + List elements; + if (this.type == TokenType.comma + || this.type == TokenType.bracketR + || this.type == TokenType.ellipsis) { + elements = this.parseExprList(TokenType.bracketR, true, true, refDestructuringErrors); + } else { + Expression firstExpr = this.parseMaybeAssign(false, refDestructuringErrors, null); + // check whether this is a postfix array comprehension + if (this.type == TokenType._for || this.type == TokenType._if) { + ComprehensionExpression c = this.parseComprehension(startLoc, false, firstExpr); + this.expect(TokenType.bracketR); + return this.finishNode(c); + } else { + this.eat(TokenType.comma); + elements = new ArrayList(); + elements.add(firstExpr); + elements.addAll( + this.parseExprList(TokenType.bracketR, true, true, refDestructuringErrors)); + } + } + return this.finishNode(new ArrayExpression(new SourceLocation(startLoc), elements)); + } else if (options.v8Extensions() && this.type == TokenType.modulo) { + // parse V8 native + this.next(); + Identifier name = this.parseIdent(true); + this.expect(TokenType.parenL); + List args = this.parseExprList(TokenType.parenR, false, false, null); + CallExpression node = + new CallExpression( + new SourceLocation(startLoc), name, new ArrayList<>(), args, false, false); + return this.finishNode(node); + } else if (options.e4x() && this.type == at) { + // this could be either a decorator or an attribute selector; we first + // try parsing it as a decorator, and then convert it to an attribute selector + // if the next token turns out not to be `class` + List decorators = parseDecorators(); + Expression attr = null; + if (decorators.size() > 1 + || this.type == TokenType._class + || ((attr = decoratorToAttributeSelector(decorators.get(0))) == null)) { + ClassExpression ce = (ClassExpression) this.parseClass(startLoc, false); + ce.addDecorators(decorators); + return ce; + } + return attr; + } else { + return super.parseExprAtom(refDestructuringErrors); + } + } + + protected Node parseLetExpression(Position startLoc, boolean maybeStatement) { + // this method assumes that the keyword `let` and the opening parenthesis have already been + // consumed + VariableDeclaration decl = this.parseVar(startLoc, false, "let"); + this.expect(TokenType.parenR); + + if (this.type == TokenType.braceL) { + if (!maybeStatement) { + // must be the start of an object literal + Expression body = this.parseObj(false, null); + return this.finishNode( + new LetExpression(new SourceLocation(startLoc), decl.getDeclarations(), body)); + } + + BlockStatement body = this.parseBlock(false); + return this.finishNode( + new LetStatement(new SourceLocation(startLoc), decl.getDeclarations(), body)); + } else if (maybeStatement) { + Position pos = startLoc; + Statement body = this.parseStatement(true, false); + if (body == null) this.unexpected(pos); + return this.finishNode( + new LetStatement(new SourceLocation(startLoc), decl.getDeclarations(), body)); + } else { + Expression body = this.parseExpression(false, null); + return this.finishNode( + new LetExpression(new SourceLocation(startLoc), decl.getDeclarations(), body)); + } + } + + // add parsing of expression closures and JScript methods + @Override + protected INode parseFunction( + Position startLoc, boolean isStatement, boolean allowExpressionBody, boolean isAsync) { + if (isFunctionSent(isStatement)) + return super.parseFunction(startLoc, isStatement, allowExpressionBody, isAsync); + allowExpressionBody = allowExpressionBody || options.mozExtensions(); + boolean oldInGen = this.inGenerator, oldInAsync = this.inAsync; + int oldYieldPos = this.yieldPos, oldAwaitPos = this.awaitPos; + Pair p = parseFunctionName(isStatement, isAsync); + boolean generator = p.fst(); + Identifier id = p.snd(), iface = null; + if (options.jscript()) { + if (isStatement && this.eatDoubleColon()) { + iface = p.snd(); + id = this.parseIdent(false); + } + } + IFunction result = + parseFunctionRest( + startLoc, + isStatement, + allowExpressionBody, + oldInGen, + oldInAsync, + oldYieldPos, + oldAwaitPos, + generator, + id); + if (iface != null) { + /* Translate JScript double colon method declarations into normal method definitions: + * + * function A::f(...) { ... } + * + * becomes + * + * A.f = function f(...) { ... }; + */ + SourceLocation memloc = + new SourceLocation( + iface.getName() + "::" + id.getName(), + iface.getLoc().getStart(), + id.getLoc().getEnd()); + MemberExpression mem = + new MemberExpression( + memloc, iface, new Identifier(id.getLoc(), id.getName()), false, false, false); + AssignmentExpression assgn = + new AssignmentExpression( + result.getLoc(), "=", mem, ((FunctionDeclaration) result).asFunctionExpression()); + return new ExpressionStatement(result.getLoc(), assgn); + } + return result; + } + + private boolean eatDoubleColon() { + if (this.eat(TokenType.colon)) { + this.expect(TokenType.colon); + return true; + } else { + return this.eat(doubleColon); + } + } + + // accept `yield` in non-generator functions + @Override + protected Expression parseMaybeAssign( + boolean noIn, DestructuringErrors refDestructuringErrors, AfterLeftParse afterLeftParse) { + if (options.mozExtensions() && isContextual("yield")) { + if (!this.inFunction) this.raise(this.startLoc, "Yield not in function"); + return this.parseYield(); + } + return super.parseMaybeAssign(noIn, refDestructuringErrors, afterLeftParse); + } + + // add parsing of comprehensions + protected ComprehensionExpression parseComprehension( + Position startLoc, boolean isGenerator, Expression body) { + List blocks = new ArrayList(); + while (this.type == TokenType._for) { + SourceLocation blockStart = new SourceLocation(this.startLoc); + boolean of = false; + this.next(); + if (this.eatContextual("each")) of = true; + this.expect(TokenType.parenL); + Expression left = this.parseBindingAtom(); + this.checkLVal(left, true, null); + if (this.eatContextual("of")) { + of = true; + } else { + this.expect(TokenType._in); + } + Expression right = this.parseExpression(false, null); + this.expect(TokenType.parenR); + blocks.add(this.finishNode(new ComprehensionBlock(blockStart, (IPattern) left, right, of))); + } + Expression filter = this.eat(TokenType._if) ? this.parseParenExpression() : null; + if (body == null) body = this.parseExpression(false, null); + + return new ComprehensionExpression( + new SourceLocation(startLoc), body, blocks, filter, isGenerator); + } + + @Override + protected Expression parseParenAndDistinguishExpression(boolean canBeArrow) { + if (options.mozExtensions()) { + // check whether next token is `for`, suggesting a generator comprehension + Position startLoc = this.startLoc; + Matcher m = Whitespace.skipWhiteSpace.matcher(this.input); + if (m.find(this.pos)) { + if (m.end() + 3 < input.length() + && "for".equals(input.substring(m.end(), m.end() + 3)) + && !Identifiers.isIdentifierChar(input.charAt(m.end() + 3), true)) { + next(); + ComprehensionExpression c = parseComprehension(startLoc, true, null); + this.expect(TokenType.parenR); + return this.finishNode(c); + } + } + } + + Expression res = super.parseParenAndDistinguishExpression(canBeArrow); + if (res instanceof ParenthesizedExpression) { + ParenthesizedExpression p = (ParenthesizedExpression) res; + if (p.getExpression() instanceof ComprehensionExpression) { + ComprehensionExpression c = (ComprehensionExpression) p.getExpression(); + if (c.isGenerator()) + return new ComprehensionExpression( + p.getLoc(), c.getBody(), c.getBlocks(), c.getFilter(), c.isGenerator()); + } + } + return res; + } + + @Override + protected boolean parseParenthesisedExpression( + DestructuringErrors refDestructuringErrors, + boolean allowTrailingComma, + ParenthesisedExpressions parenExprs, + boolean first) { + boolean cont = + super.parseParenthesisedExpression( + refDestructuringErrors, allowTrailingComma, parenExprs, first); + if (options.mozExtensions() && parenExprs.exprList.size() == 1 && this.type == TokenType._for) { + Expression body = parenExprs.exprList.remove(0); + ComprehensionExpression c = parseComprehension(body.getLoc().getStart(), true, body); + parenExprs.exprList.add(this.finishNode(c)); + return false; + } + return cont; + } + + // add parsing of for-each loops + @Override + protected Statement parseForStatement(Position startLoc) { + boolean each = false; + if (options.mozExtensions() && this.isContextual("each")) { + this.next(); + each = true; + } + Position afterEach = this.startLoc; + Statement result = super.parseForStatement(startLoc); + if (each) { + if (result instanceof ForInStatement) { + ForInStatement fis = (ForInStatement) result; + result = + new ForInStatement(fis.getLoc(), fis.getLeft(), fis.getRight(), fis.getBody(), true); + } else { + raise(afterEach, "Bad for-each statement."); + } + } + return result; + } + + // add parsing of Rhino/Nashorn-style `new` expressions with last argument after `)` + @Override + protected Expression parseNew() { + Expression res = super.parseNew(); + if (res instanceof NewExpression + && options.mozExtensions() + && !canInsertSemicolon() + && this.type == TokenType.braceL) { + ((NewExpression) res).getArguments().add(this.parseObj(false, null)); + res = this.finishNode(res); + } + return res; + } + + /* + * E4X + * + * PrimaryExpression : + * PropertyIdentifier + * XMLInitialiser + * XMLListInitialiser + * + * PropertyIdentifier : + * AttributeIdentifier + * QualifiedIdentifier + * WildcardIdent + * + * AttributeIdentifier : + * @ PropertySelector + * @ QualifiedIdentifier + * @ [ Expression ] + * + * PropertySelector : + * Identifier + * WildcardIdentifier + * + * QualifiedIdentifier : + * PropertySelector :: PropertySelector + * PropertySelector :: [ Expression ] + * + * WildcardIdentifier : + * * + * + * MemberExpression : + * MemberExpression . PropertyIdentifier + * MemberExpression .. Identifier + * MemberExpression .. PropertyIdentifier + * MemberExpression . ( Expression ) + * + * DefaultXMLNamespaceStatement : + * default xml namespace = Expression + */ + + protected TokenType doubleDot = new TokenType(new Properties(":").beforeExpr()); + + @Override + protected Token getTokenFromCode(int code) { + if (options.e4x() + && code == '.' + && charAt(this.pos + 1) == '.' + && charAt(this.pos + 2) != '.') { + this.pos += 2; + return this.finishToken(doubleDot); + } + return super.getTokenFromCode(code); + } + + // add parsing of E4X property, attribute and descendant accesses, as well as filter expressions + @Override + protected Pair parseSubscript( + Expression base, Position startLoc, boolean noCalls) { + if (options.e4x() && this.eat(TokenType.dot)) { + SourceLocation start = new SourceLocation(startLoc); + if (this.eat(TokenType.parenL)) { + Expression filter = parseExpression(false, null); + this.expect(TokenType.parenR); + return Pair.make(this.finishNode(new XMLFilterExpression(start, base, filter)), true); + } + + Expression property = this.parsePropertyIdentifierOrIdentifier(); + MemberExpression node = + new MemberExpression(start, base, property, false, false, isOnOptionalChain(false, base)); + return Pair.make(this.finishNode(node), true); + } else if (this.eat(doubleDot)) { + SourceLocation start = new SourceLocation(startLoc); + Expression property = this.parsePropertyIdentifierOrIdentifier(); + return Pair.make(this.finishNode(new XMLDotDotExpression(start, base, property)), true); + } + return super.parseSubscript(base, startLoc, noCalls); + } + + /** + * Parse a an attribute identifier, a wildcard identifier, a qualified identifier, or a plain + * identifier. + */ + protected Expression parsePropertyIdentifierOrIdentifier() { + Position start = this.startLoc; + if (this.eat(at)) { + // attribute identifier + return parseAttributeIdentifier(new SourceLocation(start)); + } else { + return parsePropertySelector(new SourceLocation(startLoc)); + } + } + + /** Parse a property selector, that is, either a wildcard identifier or a plain identifier. */ + protected Expression parsePropertySelector(SourceLocation start) { + Expression res; + if (this.eat(TokenType.star)) { + // wildcard identifier + res = this.finishNode(new XMLAnyName(start)); + } else { + res = this.parseIdent(true); + } + return res; + } + + /** + * Parse an attribute identifier, either computed ({@code [ Expr ]}) or a possibly qualified + * identifier. + */ + protected Expression parseAttributeIdentifier(SourceLocation start) { + if (this.eat(TokenType.bracketL)) { + Expression idx = parseExpression(false, null); + this.expect(TokenType.bracketR); + return this.finishNode(new XMLAttributeSelector(start, idx, true)); + } else { + return this.finishNode( + new XMLAttributeSelector( + start, parsePropertySelector(new SourceLocation(startLoc)), false)); + } + } + + @Override + protected Expression parseDecoratorBody() { + SourceLocation start = new SourceLocation(startLoc); + if (options.e4x() && this.eat(TokenType.bracketL)) { + // this must be an attribute selector, so only allow a single expression + // followed by a right bracket, which will later be converted by + // `decoratorToAttributeSelector` below + List elements = new ArrayList<>(); + elements.add(parseExpression(false, null)); + this.expect(TokenType.bracketR); + return this.finishNode(new ArrayExpression(start, elements)); + } + + return super.parseDecoratorBody(); + } + + /** + * Convert a decorator that resulted from mis-parsing an attribute selector into an attribute + * selector. + */ + protected XMLAttributeSelector decoratorToAttributeSelector(Decorator d) { + Expression e = d.getExpression(); + if (e instanceof ArrayExpression) { + ArrayExpression ae = (ArrayExpression) e; + if (ae.getElements().size() == 1) + return new XMLAttributeSelector(d.getLoc(), ae.getElements().get(0), true); + } else if (e instanceof Identifier) { + return new XMLAttributeSelector(d.getLoc(), e, false); + } + return null; + } + + @Override + protected Token readToken(int code) { + // skip XML processing instructions (which are allowed in E4X, but not in JSX); + // there is a lexical ambiguity between an XML processing instruction starting a + // chunk of E4X content and a Flow type annotation (both can start with `` of a putative XML processing instruction + // we backtrack and try lexing as something else + // to avoid frequent backtracking, we only consider `` processing instructions; + // while other processing instructions are technically possible, they are unlikely in practice + if (this.options.e4x()) { + while (code == '<') { + if (inputSubstring(this.pos + 1, this.pos + 5).equals("?xml")) { + int oldPos = this.pos; + this.pos += 5; + if (!jsx_readUntil("?>")) { + // didn't find a closing `?>`, so backtrack + this.pos = oldPos; + break; + } + } else { + break; + } + this.skipSpace(); + code = this.fullCharCodeAtPos(); + } + } + return super.readToken(code); + } + + @Override + protected Either jsx_readChunk(StringBuilder out, int chunkStart, int ch) { + // skip XML comments, processing instructions and CDATA (which are allowed in E4X, + // but not in JSX) + // unlike in `readToken` above, we know that we're inside JSX/E4X code, so there is + // no ambiguity with Flow type annotations + if (this.options.e4x() && ch == '<') { + if (inputSubstring(this.pos + 1, this.pos + 4).equals("!--")) { + out.append(inputSubstring(chunkStart, this.pos)); + this.pos += 4; + jsx_readUntil("-->"); + return Either.left(this.pos); + } else if (charAt(this.pos + 1) == '?') { + out.append(inputSubstring(chunkStart, this.pos)); + this.pos += 2; + jsx_readUntil("?>"); + return Either.left(this.pos); + } else if (inputSubstring(this.pos + 1, this.pos + 9).equals("![CDATA[")) { + out.append(inputSubstring(chunkStart, this.pos)); + this.pos += 9; + int cdataStart = this.pos; + jsx_readUntil("]]>"); + out.append(inputSubstring(cdataStart, this.pos - 3)); + return Either.left(this.pos); + } + } + + return super.jsx_readChunk(out, chunkStart, ch); + } + + private boolean jsx_readUntil(String terminator) { + char fst = terminator.charAt(0); + while (this.pos + terminator.length() <= this.input.length()) { + if (charAt(this.pos) == fst + && inputSubstring(this.pos, this.pos + terminator.length()).equals(terminator)) { + this.pos += terminator.length(); + return true; + } + ++this.pos; + } + return false; + } } diff --git a/javascript/extractor/src/com/semmle/jcorn/ESNextParser.java b/javascript/extractor/src/com/semmle/jcorn/ESNextParser.java index df6eec3df8fe..ca1a748bf075 100644 --- a/javascript/extractor/src/com/semmle/jcorn/ESNextParser.java +++ b/javascript/extractor/src/com/semmle/jcorn/ESNextParser.java @@ -1,10 +1,5 @@ package com.semmle.jcorn; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; - import com.semmle.jcorn.TokenType.Properties; import com.semmle.jcorn.jsx.JSXParser; import com.semmle.js.ast.BindExpression; @@ -42,457 +37,460 @@ import com.semmle.js.ast.Token; import com.semmle.util.collections.CollectionUtil; import com.semmle.util.data.Pair; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; /** - * An extension of the {@link JSXParser} with support for various - * unfinished ECMAScript proposals that are not supported by - * Acorn/jcorn yet. + * An extension of the {@link JSXParser} with support for various unfinished ECMAScript proposals + * that are not supported by Acorn/jcorn yet. * - * Once support becomes available, they should be removed from - * this class. + *

      Once support becomes available, they should be removed from this class. */ public class ESNextParser extends JSXParser { - public ESNextParser(Options options, String input, int startPos) { - super(options.allowImportExportEverywhere(true), input, startPos); - } - - /* - * Support for proposed language feature "Object Rest/Spread Properties" - * (http://sebmarkbage.github.io/ecmascript-rest-spread/). - */ - - @Override - protected Property parseProperty(boolean isPattern, DestructuringErrors refDestructuringErrors, - Map propHash) { - Position start = this.startLoc; - - List decorators = parseDecorators(); - - Property prop = null; - if (this.type == TokenType.ellipsis) { - SpreadElement spread = this.parseSpread(null); - Expression val; - if (isPattern) - val = new RestElement(spread.getLoc(), spread.getArgument()); - else - val = spread; - prop = this.finishNode(new Property(new SourceLocation(start), null, val, Property.Kind.INIT.name(), false, false)); - } - - if (prop == null) - prop = super.parseProperty(isPattern, refDestructuringErrors, propHash); - - prop.addDecorators(decorators); - - return prop; - } - - @Override - protected INode toAssignable(INode node, boolean isBinding) { - if (node instanceof SpreadElement) - return new RestElement(node.getLoc(), ((SpreadElement) node).getArgument()); - return super.toAssignable(node, isBinding); - } - - @Override - protected void checkLVal(INode expr, boolean isBinding, Set checkClashes) { - super.checkLVal(expr, isBinding, checkClashes); - if (expr instanceof ObjectPattern) { - ObjectPattern op = (ObjectPattern) expr; - if (op.hasRest()) - checkLVal(op.getRest(), isBinding, checkClashes); - } - } - - /* - * Support for proposed language feature "Public Class Fields" - * (http://jeffmo.github.io/es-class-public-fields/). - */ - - private boolean classProperties() { - return options.esnext(); - } - - @Override - protected MemberDefinition parseClassPropertyBody(PropertyInfo pi, - boolean hadConstructor, boolean isStatic) { - if (classProperties() && !pi.isGenerator && this.isClassProperty()) - return this.parseFieldDefinition(pi, isStatic); - return super.parseClassPropertyBody(pi, hadConstructor, isStatic); - } - - protected boolean isClassProperty() { - return this.type == TokenType.eq || this.type == TokenType.semi || this.canInsertSemicolon(); - } - - protected FieldDefinition parseFieldDefinition(PropertyInfo pi, boolean isStatic) { - Expression value = null; - if (this.type == TokenType.eq) { - this.next(); - boolean oldInFunc = this.inFunction; - this.inFunction = true; - value = parseMaybeAssign(false, null, null); - this.inFunction = oldInFunc; - } - this.semicolon(); - int flags = DeclarationFlags.getStatic(isStatic) | DeclarationFlags.getComputed(pi.computed); - return this.finishNode(new FieldDefinition(new SourceLocation(pi.startLoc), flags, pi.key, value)); - } - - /* - * Support for proposed language feature "Generator function.sent Meta Property" - * (https://github.com/allenwb/ESideas/blob/master/Generator%20metaproperty.md) - */ - private boolean functionSent() { - return options.esnext(); - } - - @Override - protected INode parseFunction(Position startLoc, boolean isStatement, - boolean allowExpressionBody, boolean isAsync) { - if (isFunctionSent(isStatement)) { - Identifier meta = this.finishNode(new Identifier(new SourceLocation(startLoc), "function")); - this.next(); - Identifier property = parseIdent(true); - if (!property.getName().equals("sent")) - this.raiseRecoverable(property, "The only valid meta property for function is function.sent"); - return this.finishNode(new MetaProperty(new SourceLocation(startLoc), meta, property)); - } - - return super.parseFunction(startLoc, isStatement, allowExpressionBody, isAsync); - } - - protected boolean isFunctionSent(boolean isStatement) { - return functionSent() && !isStatement && inGenerator && !inAsync && this.type == TokenType.dot; - } - - /* - * Support for proposed language feature "Class and Property Decorators" - * (https://github.com/wycats/javascript-decorators) - */ - private boolean decorators() { - return options.esnext(); - } - - protected TokenType at = new TokenType(new Properties("@").beforeExpr()); - - @Override - protected Token getTokenFromCode(int code) { - if (decorators() && code == 64) { - ++this.pos; - return this.finishToken(at); - } - if (functionBind() && code == 58 && charAt(this.pos+1) == 58) { - this.pos += 2; - return this.finishToken(doubleColon); - } - return super.getTokenFromCode(code); - } - - @Override - protected Statement parseStatement(boolean declaration, boolean topLevel, - Set exports) { - List decorators = this.parseDecorators(); - Statement stmt = super.parseStatement(declaration, topLevel, exports); - - if (!decorators.isEmpty()) { - if (stmt instanceof ExportDeclaration) { - Node exported = null; - if (stmt instanceof ExportDefaultDeclaration) { - exported = ((ExportDefaultDeclaration) stmt).getDeclaration(); - } else if (stmt instanceof ExportNamedDeclaration) { - exported = ((ExportNamedDeclaration) stmt).getDeclaration(); - } - if (exported instanceof ClassDeclaration) { - ((ClassDeclaration) exported).addDecorators(decorators); - } else if (exported instanceof ClassExpression) { - ((ClassExpression) exported).addDecorators(decorators); - } else { - this.raise(stmt, "Decorators can only be attached to class exports"); - } - } else if (stmt instanceof ClassDeclaration) { - ((ClassDeclaration) stmt).addDecorators(decorators); - } else if (stmt != null) { - this.raise(stmt, "Leading decorators must be attached to a class declaration"); - } - } - - return stmt; - } - - @Override - protected Expression parseExprAtom(DestructuringErrors refDestructuringErrors) { - if (this.type == at) { - List decorators = parseDecorators(); - ClassExpression ce = (ClassExpression) this.parseClass(startLoc, false); - ce.addDecorators(decorators); - return ce; - } - if (this.type == doubleColon) { - SourceLocation startLoc = new SourceLocation(this.startLoc); - this.next(); - int innerStart = this.start; - Position innerStartLoc = this.startLoc; - Expression callee = parseSubscripts(parseExprAtom(null), innerStart, innerStartLoc, true); - if (!(callee instanceof MemberExpression)) - this.raiseRecoverable(callee, "Binding should be performed on a member expression."); - return this.finishNode(new BindExpression(startLoc, null, callee)); - } - if (this.type == TokenType._import) { - Position startLoc = this.startLoc; - this.next(); - this.expect(TokenType.parenL); - return parseDynamicImport(startLoc); - } - return super.parseExprAtom(refDestructuringErrors); - } - - @Override - protected MemberDefinition parseClassMember(boolean hadConstructor) { - List decorators = parseDecorators(); - MemberDefinition member = super.parseClassMember(hadConstructor); - if (!decorators.isEmpty() && member.isConstructor()) - this.raiseRecoverable(member, "Decorators cannot be attached to class constructors."); - member.addDecorators(decorators); - return member; - } - - public List parseDecorators() { - List result = new ArrayList(); - while (this.type == at) - result.add(this.parseDecorator()); - return result; - } - - private Decorator parseDecorator() { - Position start = startLoc; - this.next(); - Expression body = parseDecoratorBody(); - Decorator decorator = new Decorator(new SourceLocation(start), body); - return this.finishNode(decorator); - } - - protected Expression parseDecoratorBody() { - Expression base; - int startPos = this.start; - Position startLoc = this.startLoc; - if (this.type == TokenType.parenL) { - base = parseParenExpression(); - } else { - base = parseIdent(true); - } - return parseSubscripts(base, startPos, startLoc, false); - } - - /* - * Support for proposed extensions to `export` - * (http://leebyron.com/ecmascript-export-ns-from and http://leebyron.com/ecmascript-export-default-from) - */ - private boolean exportExtensions() { - return options.esnext(); - } - - @Override - protected ExportDeclaration parseExportRest(SourceLocation exportStart, Set exports) { - if (exportExtensions() && this.isExportDefaultSpecifier()) { - Position specStart = this.startLoc; - Identifier exported = this.parseIdent(true); - ExportDefaultSpecifier defaultSpec = this.finishNode(new ExportDefaultSpecifier(new SourceLocation(specStart), exported)); - List specifiers = CollectionUtil.makeList(defaultSpec); - if (this.type == TokenType.comma && this.lookahead(1, true).equals("*")) { - this.next(); - specStart = this.startLoc; - this.expect(TokenType.star); - this.expectContextual("as"); - exported = this.parseIdent(false); - ExportNamespaceSpecifier nsSpec = this.finishNode(new ExportNamespaceSpecifier(new SourceLocation(specStart), exported)); - specifiers.add(nsSpec); - } else { - this.parseExportSpecifiersMaybe(specifiers, exports); - } - Literal source = (Literal) this.parseExportFrom(specifiers, null, true); - return this.finishNode(new ExportNamedDeclaration(exportStart, null, specifiers, source)); - } - - return super.parseExportRest(exportStart, exports); - } - - @Override - protected ExportDeclaration parseExportAll(SourceLocation exportStart, Position starLoc, Set exports) { - if (exportExtensions() && this.eatContextual("as")) { - Identifier exported = this.parseIdent(false); - ExportNamespaceSpecifier nsSpec = this.finishNode(new ExportNamespaceSpecifier(new SourceLocation(starLoc), exported)); - List specifiers = CollectionUtil.makeList(nsSpec); - this.parseExportSpecifiersMaybe(specifiers, exports); - Literal source = (Literal) this.parseExportFrom(specifiers, null, true); - return this.finishNode(new ExportNamedDeclaration(exportStart, null, specifiers, source)); - } - - return super.parseExportAll(exportStart, starLoc, exports); - } - - private boolean isExportDefaultSpecifier() { - if (this.type == TokenType.name) { - return !this.value.equals("type") && - !this.value.equals("async") && - !this.value.equals("interface") && - !this.value.equals("let"); - } - - if (this.type != TokenType._default) - return false; - - return this.lookahead(1, true).equals(",") || this.lookaheadIsIdent("from", true); - } - - private void parseExportSpecifiersMaybe(List specifiers, Set exports) { - if (this.eat(TokenType.comma)) { - specifiers.addAll(this.parseExportSpecifiers(exports)); - } - } - - /* - * Support for proposed language feature "Function Bind Syntax" - * (https://github.com/tc39/proposal-bind-operator) - */ - private boolean functionBind() { - return options.esnext(); - } - - protected TokenType doubleColon = new TokenType(new Properties("::").beforeExpr()); - - @Override - protected Pair parseSubscript(Expression base, Position startLoc, boolean noCalls) { - if (!noCalls && this.eat(doubleColon)) { - Expression callee = parseSubscripts(parseExprAtom(null), this.start, this.startLoc, true); - BindExpression bind = new BindExpression(new SourceLocation(startLoc), base, callee); - return Pair.make(this.finishNode(bind), true); - } - return super.parseSubscript(base, startLoc, noCalls); - } - - /* - * Support for proposed language feature "Optional Catch Binding" - * (https://github.com/tc39/proposal-optional-catch-binding) - */ - @Override - protected CatchClause parseCatchClause(Position startLoc) { - this.next(); - Expression param = null; - if (this.eat(TokenType.parenL)) { - param = this.parseBindingAtom(); - this.checkLVal(param, true, null); - this.expect(TokenType.parenR); - } else if (!options.esnext()) { - this.unexpected(); - } - BlockStatement catchBody = this.parseBlock(false); - return this.finishNode(new CatchClause(new SourceLocation(startLoc), (IPattern)param, null, catchBody)); - } - - /* - * Support for proposed language feature "Dynamic import" - * (https://github.com/tc39/proposal-dynamic-import). - */ - @Override - protected Statement parseImport(Position startLoc) { - if (!options.esnext()) - return super.parseImport(startLoc); - - int startPos = this.start; - SourceLocation loc = new SourceLocation(startLoc); - this.next(); - if (this.eat(TokenType.parenL)) { - DynamicImport di = parseDynamicImport(startLoc); - Expression expr = parseSubscripts(di, startPos, startLoc, false); - return parseExpressionStatement(false, startLoc, expr); - } else { - return super.parseImportRest(loc); - } - } - - /** - * Parses a dynamic import, assuming that the keyword `import` and the - * opening parenthesis have already been consumed. - */ - private DynamicImport parseDynamicImport(Position startLoc) { - Expression source = parseMaybeAssign(false, null, null); - this.expect(TokenType.parenR); - DynamicImport di = this.finishNode(new DynamicImport(new SourceLocation(startLoc), source)); - return di; - } - - /* - * Support for proposed language feature "Asynchronous iteration" - * (https://github.com/tc39/proposal-async-iteration) - */ - @Override - protected Statement parseForStatement(Position startLoc) { - int startPos = this.start; - boolean isAwait = false; - if (this.inAsync && this.eatContextual("await")) - isAwait = true; - Statement forStmt = super.parseForStatement(startLoc); - if (isAwait) { - if (forStmt instanceof ForOfStatement) - ((ForOfStatement) forStmt).setAwait(true); - else - this.raiseRecoverable(startPos, "Only for-of statements can be annotated with 'await'."); - } - return forStmt; - } - - @Override - protected boolean parseGeneratorMarker(boolean isAsync) { - // always allow `*`, even if `isAsync` is true - return this.eat(TokenType.star); - } - - /* - * Support for proposed language feature "Numeric separators" - * (https://github.com/tc39/proposal-numeric-separator) - */ - - @Override - protected Number readInt(int radix, Integer len) { - // implementation mostly copied from super class - int start = this.pos, code = -1; - double total = 0; - // no leading underscore - boolean underscoreAllowed = false; - - for (int i = 0, e = len == null ? Integer.MAX_VALUE : len; i < e; ++i) { - if (this.pos >= this.input.length()) - break; - code = this.input.charAt(this.pos); - - if (code == '_') { - if (underscoreAllowed) { - // no adjacent underscores - underscoreAllowed = false; - ++this.pos; - continue; - } - } else { - underscoreAllowed = true; - } - - int val; - if (code >= 97) val = code - 97 + 10; // a - else if (code >= 65) val = code - 65 + 10; // A - else if (code >= 48 && code <= 57) val = code - 48; // 0-9 - else val = Integer.MAX_VALUE; - if (val >= radix) break; - - ++this.pos; - total = total * radix + val; - } - if (this.pos == start || len != null && this.pos - start != len) return null; - - if (code == '_') - // no trailing underscore - return null; - - return total; - } + public ESNextParser(Options options, String input, int startPos) { + super(options.allowImportExportEverywhere(true), input, startPos); + } + + /* + * Support for proposed language feature "Object Rest/Spread Properties" + * (http://sebmarkbage.github.io/ecmascript-rest-spread/). + */ + + @Override + protected Property parseProperty( + boolean isPattern, + DestructuringErrors refDestructuringErrors, + Map propHash) { + Position start = this.startLoc; + + List decorators = parseDecorators(); + + Property prop = null; + if (this.type == TokenType.ellipsis) { + SpreadElement spread = this.parseSpread(null); + Expression val; + if (isPattern) val = new RestElement(spread.getLoc(), spread.getArgument()); + else val = spread; + prop = + this.finishNode( + new Property( + new SourceLocation(start), null, val, Property.Kind.INIT.name(), false, false)); + } + + if (prop == null) prop = super.parseProperty(isPattern, refDestructuringErrors, propHash); + + prop.addDecorators(decorators); + + return prop; + } + + @Override + protected INode toAssignable(INode node, boolean isBinding) { + if (node instanceof SpreadElement) + return new RestElement(node.getLoc(), ((SpreadElement) node).getArgument()); + return super.toAssignable(node, isBinding); + } + + @Override + protected void checkLVal(INode expr, boolean isBinding, Set checkClashes) { + super.checkLVal(expr, isBinding, checkClashes); + if (expr instanceof ObjectPattern) { + ObjectPattern op = (ObjectPattern) expr; + if (op.hasRest()) checkLVal(op.getRest(), isBinding, checkClashes); + } + } + + /* + * Support for proposed language feature "Public Class Fields" + * (http://jeffmo.github.io/es-class-public-fields/). + */ + + private boolean classProperties() { + return options.esnext(); + } + + @Override + protected MemberDefinition parseClassPropertyBody( + PropertyInfo pi, boolean hadConstructor, boolean isStatic) { + if (classProperties() && !pi.isGenerator && this.isClassProperty()) + return this.parseFieldDefinition(pi, isStatic); + return super.parseClassPropertyBody(pi, hadConstructor, isStatic); + } + + protected boolean isClassProperty() { + return this.type == TokenType.eq || this.type == TokenType.semi || this.canInsertSemicolon(); + } + + protected FieldDefinition parseFieldDefinition(PropertyInfo pi, boolean isStatic) { + Expression value = null; + if (this.type == TokenType.eq) { + this.next(); + boolean oldInFunc = this.inFunction; + this.inFunction = true; + value = parseMaybeAssign(false, null, null); + this.inFunction = oldInFunc; + } + this.semicolon(); + int flags = DeclarationFlags.getStatic(isStatic) | DeclarationFlags.getComputed(pi.computed); + return this.finishNode( + new FieldDefinition(new SourceLocation(pi.startLoc), flags, pi.key, value)); + } + + /* + * Support for proposed language feature "Generator function.sent Meta Property" + * (https://github.com/allenwb/ESideas/blob/master/Generator%20metaproperty.md) + */ + private boolean functionSent() { + return options.esnext(); + } + + @Override + protected INode parseFunction( + Position startLoc, boolean isStatement, boolean allowExpressionBody, boolean isAsync) { + if (isFunctionSent(isStatement)) { + Identifier meta = this.finishNode(new Identifier(new SourceLocation(startLoc), "function")); + this.next(); + Identifier property = parseIdent(true); + if (!property.getName().equals("sent")) + this.raiseRecoverable( + property, "The only valid meta property for function is function.sent"); + return this.finishNode(new MetaProperty(new SourceLocation(startLoc), meta, property)); + } + + return super.parseFunction(startLoc, isStatement, allowExpressionBody, isAsync); + } + + protected boolean isFunctionSent(boolean isStatement) { + return functionSent() && !isStatement && inGenerator && !inAsync && this.type == TokenType.dot; + } + + /* + * Support for proposed language feature "Class and Property Decorators" + * (https://github.com/wycats/javascript-decorators) + */ + private boolean decorators() { + return options.esnext(); + } + + protected TokenType at = new TokenType(new Properties("@").beforeExpr()); + + @Override + protected Token getTokenFromCode(int code) { + if (decorators() && code == 64) { + ++this.pos; + return this.finishToken(at); + } + if (functionBind() && code == 58 && charAt(this.pos + 1) == 58) { + this.pos += 2; + return this.finishToken(doubleColon); + } + return super.getTokenFromCode(code); + } + + @Override + protected Statement parseStatement(boolean declaration, boolean topLevel, Set exports) { + List decorators = this.parseDecorators(); + Statement stmt = super.parseStatement(declaration, topLevel, exports); + + if (!decorators.isEmpty()) { + if (stmt instanceof ExportDeclaration) { + Node exported = null; + if (stmt instanceof ExportDefaultDeclaration) { + exported = ((ExportDefaultDeclaration) stmt).getDeclaration(); + } else if (stmt instanceof ExportNamedDeclaration) { + exported = ((ExportNamedDeclaration) stmt).getDeclaration(); + } + if (exported instanceof ClassDeclaration) { + ((ClassDeclaration) exported).addDecorators(decorators); + } else if (exported instanceof ClassExpression) { + ((ClassExpression) exported).addDecorators(decorators); + } else { + this.raise(stmt, "Decorators can only be attached to class exports"); + } + } else if (stmt instanceof ClassDeclaration) { + ((ClassDeclaration) stmt).addDecorators(decorators); + } else if (stmt != null) { + this.raise(stmt, "Leading decorators must be attached to a class declaration"); + } + } + + return stmt; + } + + @Override + protected Expression parseExprAtom(DestructuringErrors refDestructuringErrors) { + if (this.type == at) { + List decorators = parseDecorators(); + ClassExpression ce = (ClassExpression) this.parseClass(startLoc, false); + ce.addDecorators(decorators); + return ce; + } + if (this.type == doubleColon) { + SourceLocation startLoc = new SourceLocation(this.startLoc); + this.next(); + int innerStart = this.start; + Position innerStartLoc = this.startLoc; + Expression callee = parseSubscripts(parseExprAtom(null), innerStart, innerStartLoc, true); + if (!(callee instanceof MemberExpression)) + this.raiseRecoverable(callee, "Binding should be performed on a member expression."); + return this.finishNode(new BindExpression(startLoc, null, callee)); + } + if (this.type == TokenType._import) { + Position startLoc = this.startLoc; + this.next(); + this.expect(TokenType.parenL); + return parseDynamicImport(startLoc); + } + return super.parseExprAtom(refDestructuringErrors); + } + + @Override + protected MemberDefinition parseClassMember(boolean hadConstructor) { + List decorators = parseDecorators(); + MemberDefinition member = super.parseClassMember(hadConstructor); + if (!decorators.isEmpty() && member.isConstructor()) + this.raiseRecoverable(member, "Decorators cannot be attached to class constructors."); + member.addDecorators(decorators); + return member; + } + + public List parseDecorators() { + List result = new ArrayList(); + while (this.type == at) result.add(this.parseDecorator()); + return result; + } + + private Decorator parseDecorator() { + Position start = startLoc; + this.next(); + Expression body = parseDecoratorBody(); + Decorator decorator = new Decorator(new SourceLocation(start), body); + return this.finishNode(decorator); + } + + protected Expression parseDecoratorBody() { + Expression base; + int startPos = this.start; + Position startLoc = this.startLoc; + if (this.type == TokenType.parenL) { + base = parseParenExpression(); + } else { + base = parseIdent(true); + } + return parseSubscripts(base, startPos, startLoc, false); + } + + /* + * Support for proposed extensions to `export` + * (http://leebyron.com/ecmascript-export-ns-from and http://leebyron.com/ecmascript-export-default-from) + */ + private boolean exportExtensions() { + return options.esnext(); + } + + @Override + protected ExportDeclaration parseExportRest(SourceLocation exportStart, Set exports) { + if (exportExtensions() && this.isExportDefaultSpecifier()) { + Position specStart = this.startLoc; + Identifier exported = this.parseIdent(true); + ExportDefaultSpecifier defaultSpec = + this.finishNode(new ExportDefaultSpecifier(new SourceLocation(specStart), exported)); + List specifiers = CollectionUtil.makeList(defaultSpec); + if (this.type == TokenType.comma && this.lookahead(1, true).equals("*")) { + this.next(); + specStart = this.startLoc; + this.expect(TokenType.star); + this.expectContextual("as"); + exported = this.parseIdent(false); + ExportNamespaceSpecifier nsSpec = + this.finishNode(new ExportNamespaceSpecifier(new SourceLocation(specStart), exported)); + specifiers.add(nsSpec); + } else { + this.parseExportSpecifiersMaybe(specifiers, exports); + } + Literal source = (Literal) this.parseExportFrom(specifiers, null, true); + return this.finishNode(new ExportNamedDeclaration(exportStart, null, specifiers, source)); + } + + return super.parseExportRest(exportStart, exports); + } + + @Override + protected ExportDeclaration parseExportAll( + SourceLocation exportStart, Position starLoc, Set exports) { + if (exportExtensions() && this.eatContextual("as")) { + Identifier exported = this.parseIdent(false); + ExportNamespaceSpecifier nsSpec = + this.finishNode(new ExportNamespaceSpecifier(new SourceLocation(starLoc), exported)); + List specifiers = CollectionUtil.makeList(nsSpec); + this.parseExportSpecifiersMaybe(specifiers, exports); + Literal source = (Literal) this.parseExportFrom(specifiers, null, true); + return this.finishNode(new ExportNamedDeclaration(exportStart, null, specifiers, source)); + } + + return super.parseExportAll(exportStart, starLoc, exports); + } + + private boolean isExportDefaultSpecifier() { + if (this.type == TokenType.name) { + return !this.value.equals("type") + && !this.value.equals("async") + && !this.value.equals("interface") + && !this.value.equals("let"); + } + + if (this.type != TokenType._default) return false; + + return this.lookahead(1, true).equals(",") || this.lookaheadIsIdent("from", true); + } + + private void parseExportSpecifiersMaybe(List specifiers, Set exports) { + if (this.eat(TokenType.comma)) { + specifiers.addAll(this.parseExportSpecifiers(exports)); + } + } + + /* + * Support for proposed language feature "Function Bind Syntax" + * (https://github.com/tc39/proposal-bind-operator) + */ + private boolean functionBind() { + return options.esnext(); + } + + protected TokenType doubleColon = new TokenType(new Properties("::").beforeExpr()); + + @Override + protected Pair parseSubscript( + Expression base, Position startLoc, boolean noCalls) { + if (!noCalls && this.eat(doubleColon)) { + Expression callee = parseSubscripts(parseExprAtom(null), this.start, this.startLoc, true); + BindExpression bind = new BindExpression(new SourceLocation(startLoc), base, callee); + return Pair.make(this.finishNode(bind), true); + } + return super.parseSubscript(base, startLoc, noCalls); + } + + /* + * Support for proposed language feature "Optional Catch Binding" + * (https://github.com/tc39/proposal-optional-catch-binding) + */ + @Override + protected CatchClause parseCatchClause(Position startLoc) { + this.next(); + Expression param = null; + if (this.eat(TokenType.parenL)) { + param = this.parseBindingAtom(); + this.checkLVal(param, true, null); + this.expect(TokenType.parenR); + } else if (!options.esnext()) { + this.unexpected(); + } + BlockStatement catchBody = this.parseBlock(false); + return this.finishNode( + new CatchClause(new SourceLocation(startLoc), (IPattern) param, null, catchBody)); + } + + /* + * Support for proposed language feature "Dynamic import" + * (https://github.com/tc39/proposal-dynamic-import). + */ + @Override + protected Statement parseImport(Position startLoc) { + if (!options.esnext()) return super.parseImport(startLoc); + + int startPos = this.start; + SourceLocation loc = new SourceLocation(startLoc); + this.next(); + if (this.eat(TokenType.parenL)) { + DynamicImport di = parseDynamicImport(startLoc); + Expression expr = parseSubscripts(di, startPos, startLoc, false); + return parseExpressionStatement(false, startLoc, expr); + } else { + return super.parseImportRest(loc); + } + } + + /** + * Parses a dynamic import, assuming that the keyword `import` and the opening parenthesis have + * already been consumed. + */ + private DynamicImport parseDynamicImport(Position startLoc) { + Expression source = parseMaybeAssign(false, null, null); + this.expect(TokenType.parenR); + DynamicImport di = this.finishNode(new DynamicImport(new SourceLocation(startLoc), source)); + return di; + } + + /* + * Support for proposed language feature "Asynchronous iteration" + * (https://github.com/tc39/proposal-async-iteration) + */ + @Override + protected Statement parseForStatement(Position startLoc) { + int startPos = this.start; + boolean isAwait = false; + if (this.inAsync && this.eatContextual("await")) isAwait = true; + Statement forStmt = super.parseForStatement(startLoc); + if (isAwait) { + if (forStmt instanceof ForOfStatement) ((ForOfStatement) forStmt).setAwait(true); + else this.raiseRecoverable(startPos, "Only for-of statements can be annotated with 'await'."); + } + return forStmt; + } + + @Override + protected boolean parseGeneratorMarker(boolean isAsync) { + // always allow `*`, even if `isAsync` is true + return this.eat(TokenType.star); + } + + /* + * Support for proposed language feature "Numeric separators" + * (https://github.com/tc39/proposal-numeric-separator) + */ + + @Override + protected Number readInt(int radix, Integer len) { + // implementation mostly copied from super class + int start = this.pos, code = -1; + double total = 0; + // no leading underscore + boolean underscoreAllowed = false; + + for (int i = 0, e = len == null ? Integer.MAX_VALUE : len; i < e; ++i) { + if (this.pos >= this.input.length()) break; + code = this.input.charAt(this.pos); + + if (code == '_') { + if (underscoreAllowed) { + // no adjacent underscores + underscoreAllowed = false; + ++this.pos; + continue; + } + } else { + underscoreAllowed = true; + } + + int val; + if (code >= 97) val = code - 97 + 10; // a + else if (code >= 65) val = code - 65 + 10; // A + else if (code >= 48 && code <= 57) val = code - 48; // 0-9 + else val = Integer.MAX_VALUE; + if (val >= radix) break; + + ++this.pos; + total = total * radix + val; + } + if (this.pos == start || len != null && this.pos - start != len) return null; + + if (code == '_') + // no trailing underscore + return null; + + return total; + } } diff --git a/javascript/extractor/src/com/semmle/jcorn/Identifiers.java b/javascript/extractor/src/com/semmle/jcorn/Identifiers.java index 363ab88eb422..6cda9c40a88b 100644 --- a/javascript/extractor/src/com/semmle/jcorn/Identifiers.java +++ b/javascript/extractor/src/com/semmle/jcorn/Identifiers.java @@ -8,110 +8,149 @@ /// identifier.js public class Identifiers { - public static enum Dialect { - ECMA_3, ECMA_5, ECMA_6, ECMA_7, ECMA_8, STRICT, STRICT_BIND - } - - // Reserved word lists for various dialects of the language - public static final Map> reservedWords = new LinkedHashMap<>(); - static { - reservedWords.put(Dialect.ECMA_3, stringSet("abstract boolean byte char class double enum export extends final float goto implements import int interface long native package private protected public short static super synchronized throws transient volatile")); - reservedWords.put(Dialect.ECMA_5, stringSet("class enum extends super const export import")); - reservedWords.put(Dialect.ECMA_6, stringSet("enum")); - reservedWords.put(Dialect.ECMA_7, stringSet("enum")); - reservedWords.put(Dialect.ECMA_8, stringSet("enum")); - reservedWords.put(Dialect.STRICT, stringSet("implements interface let package private protected public static yield")); - reservedWords.put(Dialect.STRICT_BIND, stringSet("eval arguments")); - } - - // And the keywords - private static final String ecma5AndLessKeywords = "break case catch continue debugger default do else finally for function if return switch throw try var while with null true false instanceof typeof void delete new in this"; - private static final String ecma6Keywords = ecma5AndLessKeywords + " const class extends export import super"; - - public static final Map> keywords = new LinkedHashMap<>(); - static { - keywords.put(Dialect.ECMA_5, stringSet(ecma5AndLessKeywords)); - keywords.put(Dialect.ECMA_6, stringSet(ecma6Keywords)); - } - - private static Set stringSet(String words) { - Set result = new LinkedHashSet(); - for (String word : words.split(" ")) - result.add(word); - return result; - } - - // ## Character categories - - private static final String nonASCIIidentifierStartChars = - "\\xaa\\xb5\\xba\\xc0-\\xd6\\xd8-\\xf6\\xf8-\\u02c1\\u02c6-\\u02d1\\u02e0-\\u02e4\\u02ec\\u02ee\\u0370-\\u0374\\u0376\\u0377\\u037a-\\u037d\\u037f\\u0386\\u0388-\\u038a\\u038c\\u038e-\\u03a1\\u03a3-\\u03f5\\u03f7-\\u0481\\u048a-\\u052f\\u0531-\\u0556\\u0559\\u0561-\\u0587\\u05d0-\\u05ea\\u05f0-\\u05f2\\u0620-\\u064a\\u066e\\u066f\\u0671-\\u06d3\\u06d5\\u06e5\\u06e6\\u06ee\\u06ef\\u06fa-\\u06fc\\u06ff\\u0710\\u0712-\\u072f\\u074d-\\u07a5\\u07b1\\u07ca-\\u07ea\\u07f4\\u07f5\\u07fa\\u0800-\\u0815\\u081a\\u0824\\u0828\\u0840-\\u0858\\u08a0-\\u08b4\\u08b6-\\u08bd\\u0904-\\u0939\\u093d\\u0950\\u0958-\\u0961\\u0971-\\u0980\\u0985-\\u098c\\u098f\\u0990\\u0993-\\u09a8\\u09aa-\\u09b0\\u09b2\\u09b6-\\u09b9\\u09bd\\u09ce\\u09dc\\u09dd\\u09df-\\u09e1\\u09f0\\u09f1\\u0a05-\\u0a0a\\u0a0f\\u0a10\\u0a13-\\u0a28\\u0a2a-\\u0a30\\u0a32\\u0a33\\u0a35\\u0a36\\u0a38\\u0a39\\u0a59-\\u0a5c\\u0a5e\\u0a72-\\u0a74\\u0a85-\\u0a8d\\u0a8f-\\u0a91\\u0a93-\\u0aa8\\u0aaa-\\u0ab0\\u0ab2\\u0ab3\\u0ab5-\\u0ab9\\u0abd\\u0ad0\\u0ae0\\u0ae1\\u0af9\\u0b05-\\u0b0c\\u0b0f\\u0b10\\u0b13-\\u0b28\\u0b2a-\\u0b30\\u0b32\\u0b33\\u0b35-\\u0b39\\u0b3d\\u0b5c\\u0b5d\\u0b5f-\\u0b61\\u0b71\\u0b83\\u0b85-\\u0b8a\\u0b8e-\\u0b90\\u0b92-\\u0b95\\u0b99\\u0b9a\\u0b9c\\u0b9e\\u0b9f\\u0ba3\\u0ba4\\u0ba8-\\u0baa\\u0bae-\\u0bb9\\u0bd0\\u0c05-\\u0c0c\\u0c0e-\\u0c10\\u0c12-\\u0c28\\u0c2a-\\u0c39\\u0c3d\\u0c58-\\u0c5a\\u0c60\\u0c61\\u0c80\\u0c85-\\u0c8c\\u0c8e-\\u0c90\\u0c92-\\u0ca8\\u0caa-\\u0cb3\\u0cb5-\\u0cb9\\u0cbd\\u0cde\\u0ce0\\u0ce1\\u0cf1\\u0cf2\\u0d05-\\u0d0c\\u0d0e-\\u0d10\\u0d12-\\u0d3a\\u0d3d\\u0d4e\\u0d54-\\u0d56\\u0d5f-\\u0d61\\u0d7a-\\u0d7f\\u0d85-\\u0d96\\u0d9a-\\u0db1\\u0db3-\\u0dbb\\u0dbd\\u0dc0-\\u0dc6\\u0e01-\\u0e30\\u0e32\\u0e33\\u0e40-\\u0e46\\u0e81\\u0e82\\u0e84\\u0e87\\u0e88\\u0e8a\\u0e8d\\u0e94-\\u0e97\\u0e99-\\u0e9f\\u0ea1-\\u0ea3\\u0ea5\\u0ea7\\u0eaa\\u0eab\\u0ead-\\u0eb0\\u0eb2\\u0eb3\\u0ebd\\u0ec0-\\u0ec4\\u0ec6\\u0edc-\\u0edf\\u0f00\\u0f40-\\u0f47\\u0f49-\\u0f6c\\u0f88-\\u0f8c\\u1000-\\u102a\\u103f\\u1050-\\u1055\\u105a-\\u105d\\u1061\\u1065\\u1066\\u106e-\\u1070\\u1075-\\u1081\\u108e\\u10a0-\\u10c5\\u10c7\\u10cd\\u10d0-\\u10fa\\u10fc-\\u1248\\u124a-\\u124d\\u1250-\\u1256\\u1258\\u125a-\\u125d\\u1260-\\u1288\\u128a-\\u128d\\u1290-\\u12b0\\u12b2-\\u12b5\\u12b8-\\u12be\\u12c0\\u12c2-\\u12c5\\u12c8-\\u12d6\\u12d8-\\u1310\\u1312-\\u1315\\u1318-\\u135a\\u1380-\\u138f\\u13a0-\\u13f5\\u13f8-\\u13fd\\u1401-\\u166c\\u166f-\\u167f\\u1681-\\u169a\\u16a0-\\u16ea\\u16ee-\\u16f8\\u1700-\\u170c\\u170e-\\u1711\\u1720-\\u1731\\u1740-\\u1751\\u1760-\\u176c\\u176e-\\u1770\\u1780-\\u17b3\\u17d7\\u17dc\\u1820-\\u1877\\u1880-\\u18a8\\u18aa\\u18b0-\\u18f5\\u1900-\\u191e\\u1950-\\u196d\\u1970-\\u1974\\u1980-\\u19ab\\u19b0-\\u19c9\\u1a00-\\u1a16\\u1a20-\\u1a54\\u1aa7\\u1b05-\\u1b33\\u1b45-\\u1b4b\\u1b83-\\u1ba0\\u1bae\\u1baf\\u1bba-\\u1be5\\u1c00-\\u1c23\\u1c4d-\\u1c4f\\u1c5a-\\u1c7d\\u1c80-\\u1c88\\u1ce9-\\u1cec\\u1cee-\\u1cf1\\u1cf5\\u1cf6\\u1d00-\\u1dbf\\u1e00-\\u1f15\\u1f18-\\u1f1d\\u1f20-\\u1f45\\u1f48-\\u1f4d\\u1f50-\\u1f57\\u1f59\\u1f5b\\u1f5d\\u1f5f-\\u1f7d\\u1f80-\\u1fb4\\u1fb6-\\u1fbc\\u1fbe\\u1fc2-\\u1fc4\\u1fc6-\\u1fcc\\u1fd0-\\u1fd3\\u1fd6-\\u1fdb\\u1fe0-\\u1fec\\u1ff2-\\u1ff4\\u1ff6-\\u1ffc\\u2071\\u207f\\u2090-\\u209c\\u2102\\u2107\\u210a-\\u2113\\u2115\\u2118-\\u211d\\u2124\\u2126\\u2128\\u212a-\\u2139\\u213c-\\u213f\\u2145-\\u2149\\u214e\\u2160-\\u2188\\u2c00-\\u2c2e\\u2c30-\\u2c5e\\u2c60-\\u2ce4\\u2ceb-\\u2cee\\u2cf2\\u2cf3\\u2d00-\\u2d25\\u2d27\\u2d2d\\u2d30-\\u2d67\\u2d6f\\u2d80-\\u2d96\\u2da0-\\u2da6\\u2da8-\\u2dae\\u2db0-\\u2db6\\u2db8-\\u2dbe\\u2dc0-\\u2dc6\\u2dc8-\\u2dce\\u2dd0-\\u2dd6\\u2dd8-\\u2dde\\u3005-\\u3007\\u3021-\\u3029\\u3031-\\u3035\\u3038-\\u303c\\u3041-\\u3096\\u309b-\\u309f\\u30a1-\\u30fa\\u30fc-\\u30ff\\u3105-\\u312d\\u3131-\\u318e\\u31a0-\\u31ba\\u31f0-\\u31ff\\u3400-\\u4db5\\u4e00-\\u9fd5\\ua000-\\ua48c\\ua4d0-\\ua4fd\\ua500-\\ua60c\\ua610-\\ua61f\\ua62a\\ua62b\\ua640-\\ua66e\\ua67f-\\ua69d\\ua6a0-\\ua6ef\\ua717-\\ua71f\\ua722-\\ua788\\ua78b-\\ua7ae\\ua7b0-\\ua7b7\\ua7f7-\\ua801\\ua803-\\ua805\\ua807-\\ua80a\\ua80c-\\ua822\\ua840-\\ua873\\ua882-\\ua8b3\\ua8f2-\\ua8f7\\ua8fb\\ua8fd\\ua90a-\\ua925\\ua930-\\ua946\\ua960-\\ua97c\\ua984-\\ua9b2\\ua9cf\\ua9e0-\\ua9e4\\ua9e6-\\ua9ef\\ua9fa-\\ua9fe\\uaa00-\\uaa28\\uaa40-\\uaa42\\uaa44-\\uaa4b\\uaa60-\\uaa76\\uaa7a\\uaa7e-\\uaaaf\\uaab1\\uaab5\\uaab6\\uaab9-\\uaabd\\uaac0\\uaac2\\uaadb-\\uaadd\\uaae0-\\uaaea\\uaaf2-\\uaaf4\\uab01-\\uab06\\uab09-\\uab0e\\uab11-\\uab16\\uab20-\\uab26\\uab28-\\uab2e\\uab30-\\uab5a\\uab5c-\\uab65\\uab70-\\uabe2\\uac00-\\ud7a3\\ud7b0-\\ud7c6\\ud7cb-\\ud7fb\\uf900-\\ufa6d\\ufa70-\\ufad9\\ufb00-\\ufb06\\ufb13-\\ufb17\\ufb1d\\ufb1f-\\ufb28\\ufb2a-\\ufb36\\ufb38-\\ufb3c\\ufb3e\\ufb40\\ufb41\\ufb43\\ufb44\\ufb46-\\ufbb1\\ufbd3-\\ufd3d\\ufd50-\\ufd8f\\ufd92-\\ufdc7\\ufdf0-\\ufdfb\\ufe70-\\ufe74\\ufe76-\\ufefc\\uff21-\\uff3a\\uff41-\\uff5a\\uff66-\\uffbe\\uffc2-\\uffc7\\uffca-\\uffcf\\uffd2-\\uffd7\\uffda-\\uffdc"; - private static final String nonASCIIidentifierChars = - "\\u200c\\u200d\\xb7\\u0300-\\u036f\\u0387\\u0483-\\u0487\\u0591-\\u05bd\\u05bf\\u05c1\\u05c2\\u05c4\\u05c5\\u05c7\\u0610-\\u061a\\u064b-\\u0669\\u0670\\u06d6-\\u06dc\\u06df-\\u06e4\\u06e7\\u06e8\\u06ea-\\u06ed\\u06f0-\\u06f9\\u0711\\u0730-\\u074a\\u07a6-\\u07b0\\u07c0-\\u07c9\\u07eb-\\u07f3\\u0816-\\u0819\\u081b-\\u0823\\u0825-\\u0827\\u0829-\\u082d\\u0859-\\u085b\\u08d4-\\u08e1\\u08e3-\\u0903\\u093a-\\u093c\\u093e-\\u094f\\u0951-\\u0957\\u0962\\u0963\\u0966-\\u096f\\u0981-\\u0983\\u09bc\\u09be-\\u09c4\\u09c7\\u09c8\\u09cb-\\u09cd\\u09d7\\u09e2\\u09e3\\u09e6-\\u09ef\\u0a01-\\u0a03\\u0a3c\\u0a3e-\\u0a42\\u0a47\\u0a48\\u0a4b-\\u0a4d\\u0a51\\u0a66-\\u0a71\\u0a75\\u0a81-\\u0a83\\u0abc\\u0abe-\\u0ac5\\u0ac7-\\u0ac9\\u0acb-\\u0acd\\u0ae2\\u0ae3\\u0ae6-\\u0aef\\u0b01-\\u0b03\\u0b3c\\u0b3e-\\u0b44\\u0b47\\u0b48\\u0b4b-\\u0b4d\\u0b56\\u0b57\\u0b62\\u0b63\\u0b66-\\u0b6f\\u0b82\\u0bbe-\\u0bc2\\u0bc6-\\u0bc8\\u0bca-\\u0bcd\\u0bd7\\u0be6-\\u0bef\\u0c00-\\u0c03\\u0c3e-\\u0c44\\u0c46-\\u0c48\\u0c4a-\\u0c4d\\u0c55\\u0c56\\u0c62\\u0c63\\u0c66-\\u0c6f\\u0c81-\\u0c83\\u0cbc\\u0cbe-\\u0cc4\\u0cc6-\\u0cc8\\u0cca-\\u0ccd\\u0cd5\\u0cd6\\u0ce2\\u0ce3\\u0ce6-\\u0cef\\u0d01-\\u0d03\\u0d3e-\\u0d44\\u0d46-\\u0d48\\u0d4a-\\u0d4d\\u0d57\\u0d62\\u0d63\\u0d66-\\u0d6f\\u0d82\\u0d83\\u0dca\\u0dcf-\\u0dd4\\u0dd6\\u0dd8-\\u0ddf\\u0de6-\\u0def\\u0df2\\u0df3\\u0e31\\u0e34-\\u0e3a\\u0e47-\\u0e4e\\u0e50-\\u0e59\\u0eb1\\u0eb4-\\u0eb9\\u0ebb\\u0ebc\\u0ec8-\\u0ecd\\u0ed0-\\u0ed9\\u0f18\\u0f19\\u0f20-\\u0f29\\u0f35\\u0f37\\u0f39\\u0f3e\\u0f3f\\u0f71-\\u0f84\\u0f86\\u0f87\\u0f8d-\\u0f97\\u0f99-\\u0fbc\\u0fc6\\u102b-\\u103e\\u1040-\\u1049\\u1056-\\u1059\\u105e-\\u1060\\u1062-\\u1064\\u1067-\\u106d\\u1071-\\u1074\\u1082-\\u108d\\u108f-\\u109d\\u135d-\\u135f\\u1369-\\u1371\\u1712-\\u1714\\u1732-\\u1734\\u1752\\u1753\\u1772\\u1773\\u17b4-\\u17d3\\u17dd\\u17e0-\\u17e9\\u180b-\\u180d\\u1810-\\u1819\\u18a9\\u1920-\\u192b\\u1930-\\u193b\\u1946-\\u194f\\u19d0-\\u19da\\u1a17-\\u1a1b\\u1a55-\\u1a5e\\u1a60-\\u1a7c\\u1a7f-\\u1a89\\u1a90-\\u1a99\\u1ab0-\\u1abd\\u1b00-\\u1b04\\u1b34-\\u1b44\\u1b50-\\u1b59\\u1b6b-\\u1b73\\u1b80-\\u1b82\\u1ba1-\\u1bad\\u1bb0-\\u1bb9\\u1be6-\\u1bf3\\u1c24-\\u1c37\\u1c40-\\u1c49\\u1c50-\\u1c59\\u1cd0-\\u1cd2\\u1cd4-\\u1ce8\\u1ced\\u1cf2-\\u1cf4\\u1cf8\\u1cf9\\u1dc0-\\u1df5\\u1dfb-\\u1dff\\u203f\\u2040\\u2054\\u20d0-\\u20dc\\u20e1\\u20e5-\\u20f0\\u2cef-\\u2cf1\\u2d7f\\u2de0-\\u2dff\\u302a-\\u302f\\u3099\\u309a\\ua620-\\ua629\\ua66f\\ua674-\\ua67d\\ua69e\\ua69f\\ua6f0\\ua6f1\\ua802\\ua806\\ua80b\\ua823-\\ua827\\ua880\\ua881\\ua8b4-\\ua8c5\\ua8d0-\\ua8d9\\ua8e0-\\ua8f1\\ua900-\\ua909\\ua926-\\ua92d\\ua947-\\ua953\\ua980-\\ua983\\ua9b3-\\ua9c0\\ua9d0-\\ua9d9\\ua9e5\\ua9f0-\\ua9f9\\uaa29-\\uaa36\\uaa43\\uaa4c\\uaa4d\\uaa50-\\uaa59\\uaa7b-\\uaa7d\\uaab0\\uaab2-\\uaab4\\uaab7\\uaab8\\uaabe\\uaabf\\uaac1\\uaaeb-\\uaaef\\uaaf5\\uaaf6\\uabe3-\\uabea\\uabec\\uabed\\uabf0-\\uabf9\\ufb1e\\ufe00-\\ufe0f\\ufe20-\\ufe2f\\ufe33\\ufe34\\ufe4d-\\ufe4f\\uff10-\\uff19\\uff3f"; - - private static Pattern nonASCIIidentifierStartPattern; - private static Pattern nonASCIIidentifierPattern; - - private static Pattern nonASCIIidentifierStart() { - if (nonASCIIidentifierStartPattern == null) - nonASCIIidentifierStartPattern = Pattern.compile("[" + nonASCIIidentifierStartChars + "]"); - return nonASCIIidentifierStartPattern; - } - - private static Pattern nonASCIIidentifier() { - if (nonASCIIidentifierPattern == null) - nonASCIIidentifierPattern = Pattern.compile("[" + nonASCIIidentifierStartChars + nonASCIIidentifierChars + "]"); - return nonASCIIidentifierPattern; - } - - // These are a run-length and offset encoded representation of the - // >0xffff code points that are a valid part of identifiers. The - // offset starts at 0x10000, and each pair of numbers represents an - // offset to the next range, and then a size of the range. They were - // generated by bin/generate-identifier-regex.js - private static final int[] astralIdentifierStartCodes = {0,11,2,25,2,18,2,1,2,14,3,13,35,122,70,52,268,28,4,48,48,31,17,26,6,37,11,29,3,35,5,7,2,4,43,157,19,35,5,35,5,39,9,51,157,310,10,21,11,7,153,5,3,0,2,43,2,1,4,0,3,22,11,22,10,30,66,18,2,1,11,21,11,25,71,55,7,1,65,0,16,3,2,2,2,26,45,28,4,28,36,7,2,27,28,53,11,21,11,18,14,17,111,72,56,50,14,50,785,52,76,44,33,24,27,35,42,34,4,0,13,47,15,3,22,0,2,0,36,17,2,24,85,6,2,0,2,3,2,14,2,9,8,46,39,7,3,1,3,21,2,6,2,1,2,4,4,0,19,0,13,4,159,52,19,3,54,47,21,1,2,0,185,46,42,3,37,47,21,0,60,42,86,25,391,63,32,0,449,56,264,8,2,36,18,0,50,29,881,921,103,110,18,195,2749,1070,4050,582,8634,568,8,30,114,29,19,47,17,3,32,20,6,18,881,68,12,0,67,12,65,0,32,6124,20,754,9486,1,3071,106,6,12,4,8,8,9,5991,84,2,70,2,1,3,0,3,1,3,3,2,11,2,0,2,6,2,64,2,3,3,7,2,6,2,27,2,3,2,4,2,0,4,6,2,339,3,24,2,24,2,30,2,24,2,30,2,24,2,30,2,24,2,30,2,24,2,7,4149,196,60,67,1213,3,2,26,2,1,2,0,3,0,2,9,2,3,2,0,2,0,7,0,5,0,2,0,2,0,2,2,2,1,2,0,3,0,2,0,2,0,2,0,2,0,2,1,2,0,3,3,2,6,2,3,2,3,2,0,2,9,2,16,6,2,2,4,2,16,4421,42710,42,4148,12,221,3,5761,10591,541}; - private static final int[] astralIdentifierCodes = {509,0,227,0,150,4,294,9,1368,2,2,1,6,3,41,2,5,0,166,1,1306,2,54,14,32,9,16,3,46,10,54,9,7,2,37,13,2,9,52,0,13,2,49,13,10,2,4,9,83,11,7,0,161,11,6,9,7,3,57,0,2,6,3,1,3,2,10,0,11,1,3,6,4,4,193,17,10,9,87,19,13,9,214,6,3,8,28,1,83,16,16,9,82,12,9,9,84,14,5,9,423,9,838,7,2,7,17,9,57,21,2,13,19882,9,135,4,60,6,26,9,1016,45,17,3,19723,1,5319,4,4,5,9,7,3,6,31,3,149,2,1418,49,513,54,5,49,9,0,15,0,23,4,2,14,1361,6,2,16,3,6,2,1,2,4,2214,6,110,6,6,9,792487,239}; - - // This has a complexity linear to the value of the code. The - // assumption is that looking up astral identifier characters is - // rare. - private static boolean isInAstralSet(int code, int[] set) { - int pos = 0x10000; - for (int i = 0; i < set.length; i += 2) { - pos += set[i]; - if (pos > code) - return false; - pos += set[i + 1]; - if (pos >= code) - return true; - } - return false; - } - - // Test whether a given character code starts an identifier. - - public static boolean isIdentifierStart(int code, boolean astral) { - if (code < 65) return code == 36; - if (code < 91) return true; - if (code < 97) return code == 95; - if (code < 123) return true; - if (code <= 0xffff) return code >= 0xaa && nonASCIIidentifierStart().matcher(str(code)).matches(); - if (!astral) return false; - return isInAstralSet(code, astralIdentifierStartCodes); - } - - public static boolean isIdentifierChar(int code, boolean astral) { - if (code < 48) return code == 36; - if (code < 58) return true; - if (code < 65) return false; - if (code < 91) return true; - if (code < 97) return code == 95; - if (code < 123) return true; - if (code <= 0xffff) return code >= 0xaa && nonASCIIidentifier().matcher(str(code)).matches(); - if (!astral) return false; - return isInAstralSet(code, astralIdentifierStartCodes) || isInAstralSet(code, astralIdentifierCodes); - } - - private static String str(int i) { - return new String(Character.toChars(i)); - } + public static enum Dialect { + ECMA_3, + ECMA_5, + ECMA_6, + ECMA_7, + ECMA_8, + STRICT, + STRICT_BIND + } + + // Reserved word lists for various dialects of the language + public static final Map> reservedWords = new LinkedHashMap<>(); + + static { + reservedWords.put( + Dialect.ECMA_3, + stringSet( + "abstract boolean byte char class double enum export extends final float goto implements import int interface long native package private protected public short static super synchronized throws transient volatile")); + reservedWords.put(Dialect.ECMA_5, stringSet("class enum extends super const export import")); + reservedWords.put(Dialect.ECMA_6, stringSet("enum")); + reservedWords.put(Dialect.ECMA_7, stringSet("enum")); + reservedWords.put(Dialect.ECMA_8, stringSet("enum")); + reservedWords.put( + Dialect.STRICT, + stringSet("implements interface let package private protected public static yield")); + reservedWords.put(Dialect.STRICT_BIND, stringSet("eval arguments")); + } + + // And the keywords + private static final String ecma5AndLessKeywords = + "break case catch continue debugger default do else finally for function if return switch throw try var while with null true false instanceof typeof void delete new in this"; + private static final String ecma6Keywords = + ecma5AndLessKeywords + " const class extends export import super"; + + public static final Map> keywords = new LinkedHashMap<>(); + + static { + keywords.put(Dialect.ECMA_5, stringSet(ecma5AndLessKeywords)); + keywords.put(Dialect.ECMA_6, stringSet(ecma6Keywords)); + } + + private static Set stringSet(String words) { + Set result = new LinkedHashSet(); + for (String word : words.split(" ")) result.add(word); + return result; + } + + // ## Character categories + + private static final String nonASCIIidentifierStartChars = + "\\xaa\\xb5\\xba\\xc0-\\xd6\\xd8-\\xf6\\xf8-\\u02c1\\u02c6-\\u02d1\\u02e0-\\u02e4\\u02ec\\u02ee\\u0370-\\u0374\\u0376\\u0377\\u037a-\\u037d\\u037f\\u0386\\u0388-\\u038a\\u038c\\u038e-\\u03a1\\u03a3-\\u03f5\\u03f7-\\u0481\\u048a-\\u052f\\u0531-\\u0556\\u0559\\u0561-\\u0587\\u05d0-\\u05ea\\u05f0-\\u05f2\\u0620-\\u064a\\u066e\\u066f\\u0671-\\u06d3\\u06d5\\u06e5\\u06e6\\u06ee\\u06ef\\u06fa-\\u06fc\\u06ff\\u0710\\u0712-\\u072f\\u074d-\\u07a5\\u07b1\\u07ca-\\u07ea\\u07f4\\u07f5\\u07fa\\u0800-\\u0815\\u081a\\u0824\\u0828\\u0840-\\u0858\\u08a0-\\u08b4\\u08b6-\\u08bd\\u0904-\\u0939\\u093d\\u0950\\u0958-\\u0961\\u0971-\\u0980\\u0985-\\u098c\\u098f\\u0990\\u0993-\\u09a8\\u09aa-\\u09b0\\u09b2\\u09b6-\\u09b9\\u09bd\\u09ce\\u09dc\\u09dd\\u09df-\\u09e1\\u09f0\\u09f1\\u0a05-\\u0a0a\\u0a0f\\u0a10\\u0a13-\\u0a28\\u0a2a-\\u0a30\\u0a32\\u0a33\\u0a35\\u0a36\\u0a38\\u0a39\\u0a59-\\u0a5c\\u0a5e\\u0a72-\\u0a74\\u0a85-\\u0a8d\\u0a8f-\\u0a91\\u0a93-\\u0aa8\\u0aaa-\\u0ab0\\u0ab2\\u0ab3\\u0ab5-\\u0ab9\\u0abd\\u0ad0\\u0ae0\\u0ae1\\u0af9\\u0b05-\\u0b0c\\u0b0f\\u0b10\\u0b13-\\u0b28\\u0b2a-\\u0b30\\u0b32\\u0b33\\u0b35-\\u0b39\\u0b3d\\u0b5c\\u0b5d\\u0b5f-\\u0b61\\u0b71\\u0b83\\u0b85-\\u0b8a\\u0b8e-\\u0b90\\u0b92-\\u0b95\\u0b99\\u0b9a\\u0b9c\\u0b9e\\u0b9f\\u0ba3\\u0ba4\\u0ba8-\\u0baa\\u0bae-\\u0bb9\\u0bd0\\u0c05-\\u0c0c\\u0c0e-\\u0c10\\u0c12-\\u0c28\\u0c2a-\\u0c39\\u0c3d\\u0c58-\\u0c5a\\u0c60\\u0c61\\u0c80\\u0c85-\\u0c8c\\u0c8e-\\u0c90\\u0c92-\\u0ca8\\u0caa-\\u0cb3\\u0cb5-\\u0cb9\\u0cbd\\u0cde\\u0ce0\\u0ce1\\u0cf1\\u0cf2\\u0d05-\\u0d0c\\u0d0e-\\u0d10\\u0d12-\\u0d3a\\u0d3d\\u0d4e\\u0d54-\\u0d56\\u0d5f-\\u0d61\\u0d7a-\\u0d7f\\u0d85-\\u0d96\\u0d9a-\\u0db1\\u0db3-\\u0dbb\\u0dbd\\u0dc0-\\u0dc6\\u0e01-\\u0e30\\u0e32\\u0e33\\u0e40-\\u0e46\\u0e81\\u0e82\\u0e84\\u0e87\\u0e88\\u0e8a\\u0e8d\\u0e94-\\u0e97\\u0e99-\\u0e9f\\u0ea1-\\u0ea3\\u0ea5\\u0ea7\\u0eaa\\u0eab\\u0ead-\\u0eb0\\u0eb2\\u0eb3\\u0ebd\\u0ec0-\\u0ec4\\u0ec6\\u0edc-\\u0edf\\u0f00\\u0f40-\\u0f47\\u0f49-\\u0f6c\\u0f88-\\u0f8c\\u1000-\\u102a\\u103f\\u1050-\\u1055\\u105a-\\u105d\\u1061\\u1065\\u1066\\u106e-\\u1070\\u1075-\\u1081\\u108e\\u10a0-\\u10c5\\u10c7\\u10cd\\u10d0-\\u10fa\\u10fc-\\u1248\\u124a-\\u124d\\u1250-\\u1256\\u1258\\u125a-\\u125d\\u1260-\\u1288\\u128a-\\u128d\\u1290-\\u12b0\\u12b2-\\u12b5\\u12b8-\\u12be\\u12c0\\u12c2-\\u12c5\\u12c8-\\u12d6\\u12d8-\\u1310\\u1312-\\u1315\\u1318-\\u135a\\u1380-\\u138f\\u13a0-\\u13f5\\u13f8-\\u13fd\\u1401-\\u166c\\u166f-\\u167f\\u1681-\\u169a\\u16a0-\\u16ea\\u16ee-\\u16f8\\u1700-\\u170c\\u170e-\\u1711\\u1720-\\u1731\\u1740-\\u1751\\u1760-\\u176c\\u176e-\\u1770\\u1780-\\u17b3\\u17d7\\u17dc\\u1820-\\u1877\\u1880-\\u18a8\\u18aa\\u18b0-\\u18f5\\u1900-\\u191e\\u1950-\\u196d\\u1970-\\u1974\\u1980-\\u19ab\\u19b0-\\u19c9\\u1a00-\\u1a16\\u1a20-\\u1a54\\u1aa7\\u1b05-\\u1b33\\u1b45-\\u1b4b\\u1b83-\\u1ba0\\u1bae\\u1baf\\u1bba-\\u1be5\\u1c00-\\u1c23\\u1c4d-\\u1c4f\\u1c5a-\\u1c7d\\u1c80-\\u1c88\\u1ce9-\\u1cec\\u1cee-\\u1cf1\\u1cf5\\u1cf6\\u1d00-\\u1dbf\\u1e00-\\u1f15\\u1f18-\\u1f1d\\u1f20-\\u1f45\\u1f48-\\u1f4d\\u1f50-\\u1f57\\u1f59\\u1f5b\\u1f5d\\u1f5f-\\u1f7d\\u1f80-\\u1fb4\\u1fb6-\\u1fbc\\u1fbe\\u1fc2-\\u1fc4\\u1fc6-\\u1fcc\\u1fd0-\\u1fd3\\u1fd6-\\u1fdb\\u1fe0-\\u1fec\\u1ff2-\\u1ff4\\u1ff6-\\u1ffc\\u2071\\u207f\\u2090-\\u209c\\u2102\\u2107\\u210a-\\u2113\\u2115\\u2118-\\u211d\\u2124\\u2126\\u2128\\u212a-\\u2139\\u213c-\\u213f\\u2145-\\u2149\\u214e\\u2160-\\u2188\\u2c00-\\u2c2e\\u2c30-\\u2c5e\\u2c60-\\u2ce4\\u2ceb-\\u2cee\\u2cf2\\u2cf3\\u2d00-\\u2d25\\u2d27\\u2d2d\\u2d30-\\u2d67\\u2d6f\\u2d80-\\u2d96\\u2da0-\\u2da6\\u2da8-\\u2dae\\u2db0-\\u2db6\\u2db8-\\u2dbe\\u2dc0-\\u2dc6\\u2dc8-\\u2dce\\u2dd0-\\u2dd6\\u2dd8-\\u2dde\\u3005-\\u3007\\u3021-\\u3029\\u3031-\\u3035\\u3038-\\u303c\\u3041-\\u3096\\u309b-\\u309f\\u30a1-\\u30fa\\u30fc-\\u30ff\\u3105-\\u312d\\u3131-\\u318e\\u31a0-\\u31ba\\u31f0-\\u31ff\\u3400-\\u4db5\\u4e00-\\u9fd5\\ua000-\\ua48c\\ua4d0-\\ua4fd\\ua500-\\ua60c\\ua610-\\ua61f\\ua62a\\ua62b\\ua640-\\ua66e\\ua67f-\\ua69d\\ua6a0-\\ua6ef\\ua717-\\ua71f\\ua722-\\ua788\\ua78b-\\ua7ae\\ua7b0-\\ua7b7\\ua7f7-\\ua801\\ua803-\\ua805\\ua807-\\ua80a\\ua80c-\\ua822\\ua840-\\ua873\\ua882-\\ua8b3\\ua8f2-\\ua8f7\\ua8fb\\ua8fd\\ua90a-\\ua925\\ua930-\\ua946\\ua960-\\ua97c\\ua984-\\ua9b2\\ua9cf\\ua9e0-\\ua9e4\\ua9e6-\\ua9ef\\ua9fa-\\ua9fe\\uaa00-\\uaa28\\uaa40-\\uaa42\\uaa44-\\uaa4b\\uaa60-\\uaa76\\uaa7a\\uaa7e-\\uaaaf\\uaab1\\uaab5\\uaab6\\uaab9-\\uaabd\\uaac0\\uaac2\\uaadb-\\uaadd\\uaae0-\\uaaea\\uaaf2-\\uaaf4\\uab01-\\uab06\\uab09-\\uab0e\\uab11-\\uab16\\uab20-\\uab26\\uab28-\\uab2e\\uab30-\\uab5a\\uab5c-\\uab65\\uab70-\\uabe2\\uac00-\\ud7a3\\ud7b0-\\ud7c6\\ud7cb-\\ud7fb\\uf900-\\ufa6d\\ufa70-\\ufad9\\ufb00-\\ufb06\\ufb13-\\ufb17\\ufb1d\\ufb1f-\\ufb28\\ufb2a-\\ufb36\\ufb38-\\ufb3c\\ufb3e\\ufb40\\ufb41\\ufb43\\ufb44\\ufb46-\\ufbb1\\ufbd3-\\ufd3d\\ufd50-\\ufd8f\\ufd92-\\ufdc7\\ufdf0-\\ufdfb\\ufe70-\\ufe74\\ufe76-\\ufefc\\uff21-\\uff3a\\uff41-\\uff5a\\uff66-\\uffbe\\uffc2-\\uffc7\\uffca-\\uffcf\\uffd2-\\uffd7\\uffda-\\uffdc"; + private static final String nonASCIIidentifierChars = + "\\u200c\\u200d\\xb7\\u0300-\\u036f\\u0387\\u0483-\\u0487\\u0591-\\u05bd\\u05bf\\u05c1\\u05c2\\u05c4\\u05c5\\u05c7\\u0610-\\u061a\\u064b-\\u0669\\u0670\\u06d6-\\u06dc\\u06df-\\u06e4\\u06e7\\u06e8\\u06ea-\\u06ed\\u06f0-\\u06f9\\u0711\\u0730-\\u074a\\u07a6-\\u07b0\\u07c0-\\u07c9\\u07eb-\\u07f3\\u0816-\\u0819\\u081b-\\u0823\\u0825-\\u0827\\u0829-\\u082d\\u0859-\\u085b\\u08d4-\\u08e1\\u08e3-\\u0903\\u093a-\\u093c\\u093e-\\u094f\\u0951-\\u0957\\u0962\\u0963\\u0966-\\u096f\\u0981-\\u0983\\u09bc\\u09be-\\u09c4\\u09c7\\u09c8\\u09cb-\\u09cd\\u09d7\\u09e2\\u09e3\\u09e6-\\u09ef\\u0a01-\\u0a03\\u0a3c\\u0a3e-\\u0a42\\u0a47\\u0a48\\u0a4b-\\u0a4d\\u0a51\\u0a66-\\u0a71\\u0a75\\u0a81-\\u0a83\\u0abc\\u0abe-\\u0ac5\\u0ac7-\\u0ac9\\u0acb-\\u0acd\\u0ae2\\u0ae3\\u0ae6-\\u0aef\\u0b01-\\u0b03\\u0b3c\\u0b3e-\\u0b44\\u0b47\\u0b48\\u0b4b-\\u0b4d\\u0b56\\u0b57\\u0b62\\u0b63\\u0b66-\\u0b6f\\u0b82\\u0bbe-\\u0bc2\\u0bc6-\\u0bc8\\u0bca-\\u0bcd\\u0bd7\\u0be6-\\u0bef\\u0c00-\\u0c03\\u0c3e-\\u0c44\\u0c46-\\u0c48\\u0c4a-\\u0c4d\\u0c55\\u0c56\\u0c62\\u0c63\\u0c66-\\u0c6f\\u0c81-\\u0c83\\u0cbc\\u0cbe-\\u0cc4\\u0cc6-\\u0cc8\\u0cca-\\u0ccd\\u0cd5\\u0cd6\\u0ce2\\u0ce3\\u0ce6-\\u0cef\\u0d01-\\u0d03\\u0d3e-\\u0d44\\u0d46-\\u0d48\\u0d4a-\\u0d4d\\u0d57\\u0d62\\u0d63\\u0d66-\\u0d6f\\u0d82\\u0d83\\u0dca\\u0dcf-\\u0dd4\\u0dd6\\u0dd8-\\u0ddf\\u0de6-\\u0def\\u0df2\\u0df3\\u0e31\\u0e34-\\u0e3a\\u0e47-\\u0e4e\\u0e50-\\u0e59\\u0eb1\\u0eb4-\\u0eb9\\u0ebb\\u0ebc\\u0ec8-\\u0ecd\\u0ed0-\\u0ed9\\u0f18\\u0f19\\u0f20-\\u0f29\\u0f35\\u0f37\\u0f39\\u0f3e\\u0f3f\\u0f71-\\u0f84\\u0f86\\u0f87\\u0f8d-\\u0f97\\u0f99-\\u0fbc\\u0fc6\\u102b-\\u103e\\u1040-\\u1049\\u1056-\\u1059\\u105e-\\u1060\\u1062-\\u1064\\u1067-\\u106d\\u1071-\\u1074\\u1082-\\u108d\\u108f-\\u109d\\u135d-\\u135f\\u1369-\\u1371\\u1712-\\u1714\\u1732-\\u1734\\u1752\\u1753\\u1772\\u1773\\u17b4-\\u17d3\\u17dd\\u17e0-\\u17e9\\u180b-\\u180d\\u1810-\\u1819\\u18a9\\u1920-\\u192b\\u1930-\\u193b\\u1946-\\u194f\\u19d0-\\u19da\\u1a17-\\u1a1b\\u1a55-\\u1a5e\\u1a60-\\u1a7c\\u1a7f-\\u1a89\\u1a90-\\u1a99\\u1ab0-\\u1abd\\u1b00-\\u1b04\\u1b34-\\u1b44\\u1b50-\\u1b59\\u1b6b-\\u1b73\\u1b80-\\u1b82\\u1ba1-\\u1bad\\u1bb0-\\u1bb9\\u1be6-\\u1bf3\\u1c24-\\u1c37\\u1c40-\\u1c49\\u1c50-\\u1c59\\u1cd0-\\u1cd2\\u1cd4-\\u1ce8\\u1ced\\u1cf2-\\u1cf4\\u1cf8\\u1cf9\\u1dc0-\\u1df5\\u1dfb-\\u1dff\\u203f\\u2040\\u2054\\u20d0-\\u20dc\\u20e1\\u20e5-\\u20f0\\u2cef-\\u2cf1\\u2d7f\\u2de0-\\u2dff\\u302a-\\u302f\\u3099\\u309a\\ua620-\\ua629\\ua66f\\ua674-\\ua67d\\ua69e\\ua69f\\ua6f0\\ua6f1\\ua802\\ua806\\ua80b\\ua823-\\ua827\\ua880\\ua881\\ua8b4-\\ua8c5\\ua8d0-\\ua8d9\\ua8e0-\\ua8f1\\ua900-\\ua909\\ua926-\\ua92d\\ua947-\\ua953\\ua980-\\ua983\\ua9b3-\\ua9c0\\ua9d0-\\ua9d9\\ua9e5\\ua9f0-\\ua9f9\\uaa29-\\uaa36\\uaa43\\uaa4c\\uaa4d\\uaa50-\\uaa59\\uaa7b-\\uaa7d\\uaab0\\uaab2-\\uaab4\\uaab7\\uaab8\\uaabe\\uaabf\\uaac1\\uaaeb-\\uaaef\\uaaf5\\uaaf6\\uabe3-\\uabea\\uabec\\uabed\\uabf0-\\uabf9\\ufb1e\\ufe00-\\ufe0f\\ufe20-\\ufe2f\\ufe33\\ufe34\\ufe4d-\\ufe4f\\uff10-\\uff19\\uff3f"; + + private static Pattern nonASCIIidentifierStartPattern; + private static Pattern nonASCIIidentifierPattern; + + private static Pattern nonASCIIidentifierStart() { + if (nonASCIIidentifierStartPattern == null) + nonASCIIidentifierStartPattern = Pattern.compile("[" + nonASCIIidentifierStartChars + "]"); + return nonASCIIidentifierStartPattern; + } + + private static Pattern nonASCIIidentifier() { + if (nonASCIIidentifierPattern == null) + nonASCIIidentifierPattern = + Pattern.compile("[" + nonASCIIidentifierStartChars + nonASCIIidentifierChars + "]"); + return nonASCIIidentifierPattern; + } + + // These are a run-length and offset encoded representation of the + // >0xffff code points that are a valid part of identifiers. The + // offset starts at 0x10000, and each pair of numbers represents an + // offset to the next range, and then a size of the range. They were + // generated by bin/generate-identifier-regex.js + private static final int[] astralIdentifierStartCodes = { + 0, 11, 2, 25, 2, 18, 2, 1, 2, 14, 3, 13, 35, 122, 70, 52, 268, 28, 4, 48, 48, 31, 17, 26, 6, 37, + 11, 29, 3, 35, 5, 7, 2, 4, 43, 157, 19, 35, 5, 35, 5, 39, 9, 51, 157, 310, 10, 21, 11, 7, 153, + 5, 3, 0, 2, 43, 2, 1, 4, 0, 3, 22, 11, 22, 10, 30, 66, 18, 2, 1, 11, 21, 11, 25, 71, 55, 7, 1, + 65, 0, 16, 3, 2, 2, 2, 26, 45, 28, 4, 28, 36, 7, 2, 27, 28, 53, 11, 21, 11, 18, 14, 17, 111, 72, + 56, 50, 14, 50, 785, 52, 76, 44, 33, 24, 27, 35, 42, 34, 4, 0, 13, 47, 15, 3, 22, 0, 2, 0, 36, + 17, 2, 24, 85, 6, 2, 0, 2, 3, 2, 14, 2, 9, 8, 46, 39, 7, 3, 1, 3, 21, 2, 6, 2, 1, 2, 4, 4, 0, + 19, 0, 13, 4, 159, 52, 19, 3, 54, 47, 21, 1, 2, 0, 185, 46, 42, 3, 37, 47, 21, 0, 60, 42, 86, + 25, 391, 63, 32, 0, 449, 56, 264, 8, 2, 36, 18, 0, 50, 29, 881, 921, 103, 110, 18, 195, 2749, + 1070, 4050, 582, 8634, 568, 8, 30, 114, 29, 19, 47, 17, 3, 32, 20, 6, 18, 881, 68, 12, 0, 67, + 12, 65, 0, 32, 6124, 20, 754, 9486, 1, 3071, 106, 6, 12, 4, 8, 8, 9, 5991, 84, 2, 70, 2, 1, 3, + 0, 3, 1, 3, 3, 2, 11, 2, 0, 2, 6, 2, 64, 2, 3, 3, 7, 2, 6, 2, 27, 2, 3, 2, 4, 2, 0, 4, 6, 2, + 339, 3, 24, 2, 24, 2, 30, 2, 24, 2, 30, 2, 24, 2, 30, 2, 24, 2, 30, 2, 24, 2, 7, 4149, 196, 60, + 67, 1213, 3, 2, 26, 2, 1, 2, 0, 3, 0, 2, 9, 2, 3, 2, 0, 2, 0, 7, 0, 5, 0, 2, 0, 2, 0, 2, 2, 2, + 1, 2, 0, 3, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 1, 2, 0, 3, 3, 2, 6, 2, 3, 2, 3, 2, 0, 2, 9, 2, 16, 6, + 2, 2, 4, 2, 16, 4421, 42710, 42, 4148, 12, 221, 3, 5761, 10591, 541 + }; + private static final int[] astralIdentifierCodes = { + 509, 0, 227, 0, 150, 4, 294, 9, 1368, 2, 2, 1, 6, 3, 41, 2, 5, 0, 166, 1, 1306, 2, 54, 14, 32, + 9, 16, 3, 46, 10, 54, 9, 7, 2, 37, 13, 2, 9, 52, 0, 13, 2, 49, 13, 10, 2, 4, 9, 83, 11, 7, 0, + 161, 11, 6, 9, 7, 3, 57, 0, 2, 6, 3, 1, 3, 2, 10, 0, 11, 1, 3, 6, 4, 4, 193, 17, 10, 9, 87, 19, + 13, 9, 214, 6, 3, 8, 28, 1, 83, 16, 16, 9, 82, 12, 9, 9, 84, 14, 5, 9, 423, 9, 838, 7, 2, 7, 17, + 9, 57, 21, 2, 13, 19882, 9, 135, 4, 60, 6, 26, 9, 1016, 45, 17, 3, 19723, 1, 5319, 4, 4, 5, 9, + 7, 3, 6, 31, 3, 149, 2, 1418, 49, 513, 54, 5, 49, 9, 0, 15, 0, 23, 4, 2, 14, 1361, 6, 2, 16, 3, + 6, 2, 1, 2, 4, 2214, 6, 110, 6, 6, 9, 792487, 239 + }; + + // This has a complexity linear to the value of the code. The + // assumption is that looking up astral identifier characters is + // rare. + private static boolean isInAstralSet(int code, int[] set) { + int pos = 0x10000; + for (int i = 0; i < set.length; i += 2) { + pos += set[i]; + if (pos > code) return false; + pos += set[i + 1]; + if (pos >= code) return true; + } + return false; + } + + // Test whether a given character code starts an identifier. + + public static boolean isIdentifierStart(int code, boolean astral) { + if (code < 65) return code == 36; + if (code < 91) return true; + if (code < 97) return code == 95; + if (code < 123) return true; + if (code <= 0xffff) + return code >= 0xaa && nonASCIIidentifierStart().matcher(str(code)).matches(); + if (!astral) return false; + return isInAstralSet(code, astralIdentifierStartCodes); + } + + public static boolean isIdentifierChar(int code, boolean astral) { + if (code < 48) return code == 36; + if (code < 58) return true; + if (code < 65) return false; + if (code < 91) return true; + if (code < 97) return code == 95; + if (code < 123) return true; + if (code <= 0xffff) return code >= 0xaa && nonASCIIidentifier().matcher(str(code)).matches(); + if (!astral) return false; + return isInAstralSet(code, astralIdentifierStartCodes) + || isInAstralSet(code, astralIdentifierCodes); + } + + private static String str(int i) { + return new String(Character.toChars(i)); + } } diff --git a/javascript/extractor/src/com/semmle/jcorn/Locutil.java b/javascript/extractor/src/com/semmle/jcorn/Locutil.java index 3ef911da6e9b..901d0ef3f9cb 100644 --- a/javascript/extractor/src/com/semmle/jcorn/Locutil.java +++ b/javascript/extractor/src/com/semmle/jcorn/Locutil.java @@ -1,27 +1,24 @@ package com.semmle.jcorn; -import java.util.regex.Matcher; - import com.semmle.js.ast.Position; +import java.util.regex.Matcher; /// locutil.js public class Locutil { - /** - * The `getLineInfo` function is mostly useful when the - * `locations` option is off (for performance reasons) and you - * want to find the line/column position for a given character - * offset. `input` should be the code string that the offset refers - * into. - */ - public static Position getLineInfo(String input, int offset) { - Matcher lineBreakG = Whitespace.lineBreakG.matcher(input); - for (int line = 1, cur = 0;;) { - if (lineBreakG.find(cur) && lineBreakG.start() < offset) { - ++line; - cur = lineBreakG.end(); - } else { - return new Position(line, offset - cur, offset); - } - } - } + /** + * The `getLineInfo` function is mostly useful when the `locations` option is off (for performance + * reasons) and you want to find the line/column position for a given character offset. `input` + * should be the code string that the offset refers into. + */ + public static Position getLineInfo(String input, int offset) { + Matcher lineBreakG = Whitespace.lineBreakG.matcher(input); + for (int line = 1, cur = 0; ; ) { + if (lineBreakG.find(cur) && lineBreakG.start() < offset) { + ++line; + cur = lineBreakG.end(); + } else { + return new Position(line, offset - cur, offset); + } + } + } } diff --git a/javascript/extractor/src/com/semmle/jcorn/Options.java b/javascript/extractor/src/com/semmle/jcorn/Options.java index 417a81fec70b..63da1ee6bcb1 100644 --- a/javascript/extractor/src/com/semmle/jcorn/Options.java +++ b/javascript/extractor/src/com/semmle/jcorn/Options.java @@ -1,248 +1,259 @@ package com.semmle.jcorn; -import java.util.List; -import java.util.function.BiFunction; -import java.util.function.Function; - import com.semmle.jcorn.Identifiers.Dialect; import com.semmle.js.ast.Comment; import com.semmle.js.ast.Position; import com.semmle.js.ast.Program; import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Token; +import java.util.List; +import java.util.function.BiFunction; +import java.util.function.Function; /// options.js public class Options { - public enum AllowReserved { - YES(true), NO(false), NEVER(false); - - private final boolean isTrue; - - private AllowReserved(boolean isTrue) { - this.isTrue = isTrue; - } - - public boolean isTrue() { - return isTrue; - } - } - - public interface OnCommentCallback { - public void call(boolean block, String input, String text, int start, int end, Position startLoc, Position endLoc); - } - - private boolean allowHashBang, allowReturnOutsideFunction, allowImportExportEverywhere; - private boolean preserveParens, mozExtensions, jscript, esnext, v8Extensions, e4x; - private int ecmaVersion; - private AllowReserved allowReserved; - private String sourceType; - private BiFunction onInsertedSemicolon, onTrailingComma; - private Function onToken; - private OnCommentCallback onComment; - private Program program; - private Function onRecoverableError; - - public Options() { - this.ecmaVersion = 7; - this.sourceType = "script"; - this.onInsertedSemicolon = null; - this.onTrailingComma = null; - this.allowReserved = AllowReserved.YES; - this.allowReturnOutsideFunction = false; - this.allowImportExportEverywhere = false; - this.allowHashBang = false; - this.onToken = null; - this.onComment = null; - this.program = null; - this.preserveParens = false; - this.mozExtensions = false; - this.jscript = false; - this.esnext = false; - this.v8Extensions = false; - this.e4x = false; - this.onRecoverableError = null; - } - - public Options(Options that) { - this.allowHashBang = that.allowHashBang; - this.allowReturnOutsideFunction = that.allowReturnOutsideFunction; - this.allowImportExportEverywhere = that.allowImportExportEverywhere; - this.preserveParens = that.preserveParens; - this.mozExtensions = that.mozExtensions; - this.jscript = that.jscript; - this.esnext = that.esnext; - this.v8Extensions = that.v8Extensions; - this.e4x = that.e4x; - this.ecmaVersion = that.ecmaVersion; - this.allowReserved = that.allowReserved; - this.sourceType = that.sourceType; - this.onInsertedSemicolon = that.onInsertedSemicolon; - this.onTrailingComma = that.onTrailingComma; - this.onToken = that.onToken; - this.onComment = that.onComment; - this.program = that.program; - this.onRecoverableError = that.onRecoverableError; - } - - public boolean allowHashBang() { - return allowHashBang; - } - - public boolean allowReturnOutsideFunction() { - return allowReturnOutsideFunction; - } - - public boolean allowImportExportEverywhere() { - return allowImportExportEverywhere; - } - - public boolean preserveParens() { - return preserveParens; - } - - public boolean mozExtensions() { - return mozExtensions; - } - - public boolean jscript() { - return jscript; - } - - public boolean esnext() { - return esnext; - } - - public boolean v8Extensions() { - return v8Extensions; - } - - public boolean e4x() { - return e4x; - } - - public Identifiers.Dialect getDialect() { - switch (ecmaVersion) { - case 3: - return Dialect.ECMA_3; - case 5: - return Dialect.ECMA_5; - case 6: - return Dialect.ECMA_6; - case 8: - return Dialect.ECMA_8; - default: - return Dialect.ECMA_7; - } - } - - public int ecmaVersion() { - return ecmaVersion; - } - - public Options ecmaVersion(int ecmaVersion) { - if (ecmaVersion >= 2015) - ecmaVersion -= 2009; - - this.ecmaVersion = ecmaVersion; - if (ecmaVersion >= 5) - this.allowReserved = AllowReserved.NO; - return this; - } - - public AllowReserved allowReserved() { - return allowReserved; - } - - public Options onComment(List comments) { - this.onComment = (block, input, text, start, end, startLoc, endLoc) -> { - String src = input.substring(start, end); - comments.add(new Comment(new SourceLocation(src, startLoc, endLoc), text)); - }; - return this; - } - - public String sourceType() { - return sourceType; - } - - public Options sourceType(String sourceType) { - this.sourceType = sourceType; - return this; - } - - public Options mozExtensions(boolean mozExtensions) { - this.mozExtensions = mozExtensions; - return this; - } - - public Options jscript(boolean jscript) { - this.jscript = jscript; - return this; - } - - public Options esnext(boolean esnext) { - this.esnext = esnext; - return this; - } - - public void v8Extensions(boolean v8Extensions) { - this.v8Extensions = v8Extensions; - } - - public void e4x(boolean e4x) { - this.e4x = e4x; - } - - public Options preserveParens(boolean preserveParens) { - this.preserveParens = preserveParens; - return this; - } - - public Options allowReturnOutsideFunction(boolean allowReturnOutsideFunction) { - this.allowReturnOutsideFunction = allowReturnOutsideFunction; - return this; - } - - public Options allowImportExportEverywhere(boolean allowImportExportEverywhere) { - this.allowImportExportEverywhere = allowImportExportEverywhere; - return this; - } - - public BiFunction onInsertedSemicolon() { - return onInsertedSemicolon; - } - - public BiFunction onTrailingComma() { - return onTrailingComma; - } - - public Function onToken() { - return onToken; - } - - public Options onToken(List tokens) { - return onToken((tk) -> { tokens.add(tk); return null; }); - } - - public Options onToken(Function tmp) { - this.onToken = tmp; - return this; - } - - public OnCommentCallback onComment() { - return onComment; - } - - public Program program() { - return program; - } - - public Options onRecoverableError(Function onRecoverableError) { - this.onRecoverableError = onRecoverableError; - return this; - } - - public Function onRecoverableError() { - return onRecoverableError; - } + public enum AllowReserved { + YES(true), + NO(false), + NEVER(false); + + private final boolean isTrue; + + private AllowReserved(boolean isTrue) { + this.isTrue = isTrue; + } + + public boolean isTrue() { + return isTrue; + } + } + + public interface OnCommentCallback { + public void call( + boolean block, + String input, + String text, + int start, + int end, + Position startLoc, + Position endLoc); + } + + private boolean allowHashBang, allowReturnOutsideFunction, allowImportExportEverywhere; + private boolean preserveParens, mozExtensions, jscript, esnext, v8Extensions, e4x; + private int ecmaVersion; + private AllowReserved allowReserved; + private String sourceType; + private BiFunction onInsertedSemicolon, onTrailingComma; + private Function onToken; + private OnCommentCallback onComment; + private Program program; + private Function onRecoverableError; + + public Options() { + this.ecmaVersion = 7; + this.sourceType = "script"; + this.onInsertedSemicolon = null; + this.onTrailingComma = null; + this.allowReserved = AllowReserved.YES; + this.allowReturnOutsideFunction = false; + this.allowImportExportEverywhere = false; + this.allowHashBang = false; + this.onToken = null; + this.onComment = null; + this.program = null; + this.preserveParens = false; + this.mozExtensions = false; + this.jscript = false; + this.esnext = false; + this.v8Extensions = false; + this.e4x = false; + this.onRecoverableError = null; + } + + public Options(Options that) { + this.allowHashBang = that.allowHashBang; + this.allowReturnOutsideFunction = that.allowReturnOutsideFunction; + this.allowImportExportEverywhere = that.allowImportExportEverywhere; + this.preserveParens = that.preserveParens; + this.mozExtensions = that.mozExtensions; + this.jscript = that.jscript; + this.esnext = that.esnext; + this.v8Extensions = that.v8Extensions; + this.e4x = that.e4x; + this.ecmaVersion = that.ecmaVersion; + this.allowReserved = that.allowReserved; + this.sourceType = that.sourceType; + this.onInsertedSemicolon = that.onInsertedSemicolon; + this.onTrailingComma = that.onTrailingComma; + this.onToken = that.onToken; + this.onComment = that.onComment; + this.program = that.program; + this.onRecoverableError = that.onRecoverableError; + } + + public boolean allowHashBang() { + return allowHashBang; + } + + public boolean allowReturnOutsideFunction() { + return allowReturnOutsideFunction; + } + + public boolean allowImportExportEverywhere() { + return allowImportExportEverywhere; + } + + public boolean preserveParens() { + return preserveParens; + } + + public boolean mozExtensions() { + return mozExtensions; + } + + public boolean jscript() { + return jscript; + } + + public boolean esnext() { + return esnext; + } + + public boolean v8Extensions() { + return v8Extensions; + } + + public boolean e4x() { + return e4x; + } + + public Identifiers.Dialect getDialect() { + switch (ecmaVersion) { + case 3: + return Dialect.ECMA_3; + case 5: + return Dialect.ECMA_5; + case 6: + return Dialect.ECMA_6; + case 8: + return Dialect.ECMA_8; + default: + return Dialect.ECMA_7; + } + } + + public int ecmaVersion() { + return ecmaVersion; + } + + public Options ecmaVersion(int ecmaVersion) { + if (ecmaVersion >= 2015) ecmaVersion -= 2009; + + this.ecmaVersion = ecmaVersion; + if (ecmaVersion >= 5) this.allowReserved = AllowReserved.NO; + return this; + } + + public AllowReserved allowReserved() { + return allowReserved; + } + + public Options onComment(List comments) { + this.onComment = + (block, input, text, start, end, startLoc, endLoc) -> { + String src = input.substring(start, end); + comments.add(new Comment(new SourceLocation(src, startLoc, endLoc), text)); + }; + return this; + } + + public String sourceType() { + return sourceType; + } + + public Options sourceType(String sourceType) { + this.sourceType = sourceType; + return this; + } + + public Options mozExtensions(boolean mozExtensions) { + this.mozExtensions = mozExtensions; + return this; + } + + public Options jscript(boolean jscript) { + this.jscript = jscript; + return this; + } + + public Options esnext(boolean esnext) { + this.esnext = esnext; + return this; + } + + public void v8Extensions(boolean v8Extensions) { + this.v8Extensions = v8Extensions; + } + + public void e4x(boolean e4x) { + this.e4x = e4x; + } + + public Options preserveParens(boolean preserveParens) { + this.preserveParens = preserveParens; + return this; + } + + public Options allowReturnOutsideFunction(boolean allowReturnOutsideFunction) { + this.allowReturnOutsideFunction = allowReturnOutsideFunction; + return this; + } + + public Options allowImportExportEverywhere(boolean allowImportExportEverywhere) { + this.allowImportExportEverywhere = allowImportExportEverywhere; + return this; + } + + public BiFunction onInsertedSemicolon() { + return onInsertedSemicolon; + } + + public BiFunction onTrailingComma() { + return onTrailingComma; + } + + public Function onToken() { + return onToken; + } + + public Options onToken(List tokens) { + return onToken( + (tk) -> { + tokens.add(tk); + return null; + }); + } + + public Options onToken(Function tmp) { + this.onToken = tmp; + return this; + } + + public OnCommentCallback onComment() { + return onComment; + } + + public Program program() { + return program; + } + + public Options onRecoverableError(Function onRecoverableError) { + this.onRecoverableError = onRecoverableError; + return this; + } + + public Function onRecoverableError() { + return onRecoverableError; + } } diff --git a/javascript/extractor/src/com/semmle/jcorn/Parser.java b/javascript/extractor/src/com/semmle/jcorn/Parser.java index b99dbe0b1d53..6062c86167c0 100644 --- a/javascript/extractor/src/com/semmle/jcorn/Parser.java +++ b/javascript/extractor/src/com/semmle/jcorn/Parser.java @@ -3,19 +3,6 @@ import static com.semmle.jcorn.Whitespace.isNewLine; import static com.semmle.jcorn.Whitespace.lineBreak; -import java.io.File; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.Stack; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - import com.semmle.jcorn.Identifiers.Dialect; import com.semmle.jcorn.Options.AllowReserved; import com.semmle.js.ast.ArrayExpression; @@ -36,6 +23,7 @@ import com.semmle.js.ast.ConditionalExpression; import com.semmle.js.ast.ContinueStatement; import com.semmle.js.ast.DebuggerStatement; +import com.semmle.js.ast.DeclarationFlags; import com.semmle.js.ast.DoWhileStatement; import com.semmle.js.ast.EmptyStatement; import com.semmle.js.ast.EnhancedForStatement; @@ -65,7 +53,6 @@ import com.semmle.js.ast.LogicalExpression; import com.semmle.js.ast.MemberDefinition; import com.semmle.js.ast.MemberExpression; -import com.semmle.js.ast.DeclarationFlags; import com.semmle.js.ast.MetaProperty; import com.semmle.js.ast.MethodDefinition; import com.semmle.js.ast.NewExpression; @@ -103,3492 +90,3601 @@ import com.semmle.util.collections.CollectionUtil; import com.semmle.util.data.Pair; import com.semmle.util.data.StringUtil; -import com.semmle.util.exception.Exceptions; import com.semmle.util.exception.CatastrophicError; +import com.semmle.util.exception.Exceptions; import com.semmle.util.io.WholeIO; +import java.io.File; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.Stack; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Java port of Acorn. * - * This version corresponds to Acorn 4.0.3, - * but does not support plugins, and always tracks full source locations. + *

      This version corresponds to Acorn + * 4.0.3, but does not support plugins, and always tracks full source locations. */ public class Parser { - protected final Options options; - protected final Set keywords; - private final Set reservedWords, reservedWordsStrict, reservedWordsStrictBind; - protected final String input; - private boolean containsEsc; - protected boolean exprAllowed; - protected boolean strict; - private boolean inModule; - protected boolean inFunction; - protected boolean inGenerator; - protected boolean inAsync; - protected boolean inTemplateElement; - protected int pos; - protected int lineStart; - protected int curLine; - protected int start; - protected int end; - protected TokenType type; - protected Object value; - protected Position startLoc; - protected Position endLoc; - protected Position lastTokEndLoc, lastTokStartLoc; - protected int lastTokStart, lastTokEnd; - protected Stack context; - protected int potentialArrowAt; - private Stack labels; - protected int yieldPos, awaitPos; - - /** - * For readability purposes, we pass this instead of false as the argument to - * the hasDeclareKeyword parameter (which only exists in TypeScript). - */ - private static final boolean noDeclareKeyword = false; - - /** - * For readability purposes, we pass this instead of false as the argument to - * the isAbstract parameter (which only exists in TypeScript). - */ - protected static final boolean notAbstract = false; - - /** - * For readability purposes, we pass this instead of null as the argument to the - * type annotation parameters (which only exists in TypeScript). - */ - private static final ITypeExpression noTypeAnnotation = null; - - protected static class LabelInfo { - String name, kind; - int statementStart; - - public LabelInfo(String name, String kind, int statementStart) { - this.name = name; - this.kind = kind; - this.statementStart = statementStart; - } - } - - public static void main(String[] args) { - new Parser(new Options(), new WholeIO().strictread(new File(args[0])), 0).parse(); - } - - /// begin state.js - - public Parser(Options options, String input, int startPos) { - this.options = options; - this.keywords = new LinkedHashSet(Identifiers.keywords.get(options.ecmaVersion() >= 6 ? Identifiers.Dialect.ECMA_6 : Identifiers.Dialect.ECMA_5)); - this.reservedWords = new LinkedHashSet(); - if (!options.allowReserved().isTrue()) { - this.reservedWords.addAll(Identifiers.reservedWords.get(options.getDialect())); - if (options.sourceType().equals("module")) - this.reservedWords.add("await"); - } - this.reservedWordsStrict = new LinkedHashSet(this.reservedWords); - this.reservedWordsStrict.addAll(Identifiers.reservedWords.get(Dialect.STRICT)); - this.reservedWordsStrictBind = new LinkedHashSet(this.reservedWordsStrict); - this.reservedWordsStrictBind.addAll(Identifiers.reservedWords.get(Dialect.STRICT_BIND)); - this.input = input; - - // Used to signal to callers of `readWord1` whether the word - // contained any escape sequences. This is needed because words with - // escape sequences must not be interpreted as keywords. - this.containsEsc = false; - - // Set up token state - - // The current position of the tokenizer in the input. - if (startPos != 0) { - this.pos = startPos; - this.lineStart = this.input.lastIndexOf("\n", startPos - 1) + 1; - this.curLine = inputSubstring(0, this.lineStart).split(Whitespace.lineBreak).length; - } else { - this.pos = this.lineStart = 0; - this.curLine = 1; - } - - // Properties of the current token: - // Its type - this.type = TokenType.eof; - // For tokens that include more information than their type, the value - this.value = null; - // Its start and end offset - this.start = this.end = this.pos; - // And, if locations are used, the {line, column} object - // corresponding to those offsets - this.startLoc = this.endLoc = this.curPosition(); - - // Position information for the previous token - this.lastTokEndLoc = this.lastTokStartLoc = null; - this.lastTokStart = this.lastTokEnd = this.pos; - - // The context stack is used to superficially track syntactic - // context to predict whether a regular expression is allowed in a - // given position. - this.context = this.initialContext(); - this.exprAllowed = true; - - // Figure out if it's a module code. - this.strict = this.inModule = options.sourceType().equals("module"); - - // Used to signify the start of a potential arrow function - this.potentialArrowAt = -1; - - // Flags to track whether we are in a function, a generator, an async function. - this.inFunction = this.inGenerator = this.inAsync = false; - // Positions to delayed-check that yield/await does not exist in default parameters. - this.yieldPos = this.awaitPos = 0; - // Labels in scope. - this.labels = new Stack(); - - // If enabled, skip leading hashbang line. - if (this.pos == 0 && options.allowHashBang() && this.input.startsWith("#!")) - this.skipLineComment(2); - } - - public Program parse() { - Position startLoc = this.startLoc; - this.nextToken(); - return this.parseTopLevel(startLoc, this.options.program()); - } - - /// end state.js - - /// begin location.js - protected void raise(int pos, String msg, boolean recoverable) { - Position loc = Locutil.getLineInfo(input, pos); - raise(loc, msg, recoverable); - } - - protected void raise(int pos, String msg) { - raise(pos, msg, false); - } - - protected void raise(Position loc, String msg, boolean recoverable) { - msg += " (" + loc.getLine() + ":" + loc.getColumn() + ")"; - SyntaxError err = new SyntaxError(msg, loc, this.pos); - if (recoverable && options.onRecoverableError() != null) - options.onRecoverableError().apply(err); - else - throw err; - } - - protected void raise(Position loc, String msg) { - raise(loc, msg, false); - } - - protected void raise(INode nd, String msg) { - raise(nd.getLoc().getStart(), msg, false); - } - - protected void raiseRecoverable(int pos, String msg) { - raise(pos, msg, true); - } - - protected void raiseRecoverable(INode nd, String msg) { - raise(nd.getLoc().getStart(), msg, true); - } - - protected Position curPosition() { - return new Position(curLine, pos - lineStart, pos); - } - - /// end location.js - - /// begin tokenize.js - - // Move to the next token - - protected void next() { - if (this.options.onToken() != null) - this.options.onToken().apply(mkToken()); - - this.lastTokEnd = this.end; - this.lastTokStart = this.start; - this.lastTokEndLoc = this.endLoc; - this.lastTokStartLoc = this.startLoc; - this.nextToken(); - } - - // Toggle strict mode. Re-reads the next number or string to please - // pedantic tests (`"use strict"; 010;` should fail). - - public void setStrict(boolean strict) { - this.strict = strict; - if (this.type != TokenType.num && this.type != TokenType.string) return; - this.pos = this.start; - while (this.pos < this.lineStart) { - this.lineStart = this.input.lastIndexOf("\n", this.lineStart - 2) + 1; - --this.curLine; - } - this.nextToken(); - } - - public TokContext curContext() { - return context.peek(); - } - - // Read a single token, updating the parser object's token-related - // properties. - - public Token nextToken() { - TokContext curContext = this.curContext(); - if (curContext == null || !curContext.preserveSpace) - this.skipSpace(); - - this.start = this.pos; - this.startLoc = this.curPosition(); - if (this.pos >= this.input.length()) - return this.finishToken(TokenType.eof); - - if (curContext != null && curContext.override != null) - return curContext.override.apply(this); - else - return this.readToken(this.fullCharCodeAtPos()); - } - - protected Token readToken(int code) { - // Identifier or keyword. '\\uXXXX' sequences are allowed in - // identifiers, so '\' also dispatches to that. - if (Identifiers.isIdentifierStart(code, this.options.ecmaVersion() >= 6) || code == 92 /* '\' */) - return this.readWord(); - - return this.getTokenFromCode(code); - } - - protected int fullCharCodeAtPos() { - int code = charAt(this.pos); - if (code <= 0xd7ff || code >= 0xe000) - return code; - int next = charAt(this.pos + 1); - return (code << 10) + next - 0x35fdc00; - } - - protected void skipBlockComment() { - Position startLoc = this.options.onComment() != null ? this.curPosition() : null; - int start = this.pos, end = this.input.indexOf("*/", this.pos += 2); - if (end == -1) - this.raise(this.pos - 2, "Unterminated comment"); - this.pos = end + 2; - Matcher m = Whitespace.lineBreakG.matcher(this.input); - int next = start; - while (m.find(next) && m.start() < this.pos) { - ++this.curLine; - lineStart = m.end(); - next = lineStart; - } - if (this.options.onComment() != null) - this.options.onComment().call(true, this.input, inputSubstring(start + 2, end), start, this.pos, - startLoc, this.curPosition()); - } - - protected void skipLineComment(int startSkip) { - int start = this.pos; - Position startLoc = this.options.onComment() != null ? this.curPosition() : null; - this.pos += startSkip; - int ch = charAt(this.pos); - while (this.pos < this.input.length() && ch != 10 && ch != 13 && ch != 8232 && ch != 8233) { - ++this.pos; - ch = charAt(this.pos); - } - if (this.options.onComment() != null) - this.options.onComment().call(false, this.input, inputSubstring(start + startSkip, this.pos), start, this.pos, - startLoc, this.curPosition()); - } - - // Called at the start of the parse and after every token. Skips - // whitespace and comments, and. - - protected void skipSpace() { - loop: while (this.pos < this.input.length()) { - int ch = this.input.charAt(this.pos); - switch (ch) { - case 32: case 160: // ' ' - ++this.pos; - break; - case 13: - if (charAt(this.pos + 1) == 10) { - ++this.pos; - } - case 10: case 8232: case 8233: - ++this.pos; - ++this.curLine; - this.lineStart = this.pos; - break; - case 47: // '/' - switch (charAt(this.pos + 1)) { - case 42: // '*' - this.skipBlockComment(); - break; - case 47: - this.skipLineComment(2); - break; - default: - break loop; - } - break; - default: - if (ch > 8 && ch < 14 || ch >= 5760 && Whitespace.nonASCIIwhitespace.indexOf(ch) > -1) { - ++this.pos; - } else { - break loop; - } - } - } - } - - // Called at the end of every token. Sets `end`, `val`, and - // maintains `context` and `exprAllowed`, and skips the space after - // the token, so that the next one's `start` will point at the - // right position. - - protected Token finishToken(TokenType type, Object val) { - this.end = this.pos; - this.endLoc = this.curPosition(); - TokenType prevType = this.type; - this.type = type; - this.value = val; - this.updateContext(prevType); - return mkToken(); - } - - private Token mkToken() { - String src = inputSubstring(start, end); - SourceLocation loc = new SourceLocation(src, startLoc, endLoc); - String label, keyword; - if (isKeyword(src)) { - label = keyword = src; - } else { - label = type.label; - keyword = type.keyword; - } - return new Token(loc, label, keyword); - } - - protected boolean isKeyword(String src) { - if (type.keyword != null) - return true; - if (type == TokenType.name) { - if (keywords.contains(src)) - return true; - if (options.ecmaVersion() >= 6 && ("let".equals(src) || "yield".equals(src))) - return true; - } - return false; - } - - protected Token finishToken(TokenType type) { - return finishToken(type, null); - } - - // ### Token reading - - // This is the function that is called to fetch the next token. It - // is somewhat obscure, because it works in character codes rather - // than characters, and because operator parsing has been inlined - // into it. - // - // All in the name of speed. - // - private Token readToken_dot() { - int next = charAt(this.pos + 1); - if (next >= 48 && next <= 57) return this.readNumber(true); - int next2 = charAt(this.pos + 2); - if (this.options.ecmaVersion() >= 6 && next == 46 && next2 == 46) { // 46 = dot '.' - this.pos += 3; - return this.finishToken(TokenType.ellipsis); - } else { - ++this.pos; - return this.finishToken(TokenType.dot); - } - } - - private Token readToken_question() { // '?' - int next = charAt(this.pos + 1); - int next2 = charAt(this.pos + 2); - if (this.options.esnext()) { - if (next == '.' && !('0' <= next2 && next2 <= '9')) // '?.', but not '?.X' where X is a digit - return this.finishOp(TokenType.questiondot, 2); - if (next == '?') // '??' - return this.finishOp(TokenType.questionquestion, 2); - } - return this.finishOp(TokenType.question, 1); - } - - private Token readToken_slash() { // '/' - int next = charAt(this.pos + 1); - if (this.exprAllowed) { - ++this.pos; - return this.readRegexp(); - } - if (next == 61) - return this.finishOp(TokenType.assign, 2); - return this.finishOp(TokenType.slash, 1); - } - - private Token readToken_mult_modulo_exp(int code) { // '%*' - int next = charAt(this.pos + 1); - int size = 1; - TokenType tokentype = code == 42 ? TokenType.star : TokenType.modulo; - - // exponentiation operator ** and **= - if (this.options.ecmaVersion() >= 7 && code == 42 && next == 42) { - ++size; - tokentype = TokenType.starstar; - next = charAt(this.pos + 2); - } - - if (next == 61) - return this.finishOp(TokenType.assign, size + 1); - return this.finishOp(tokentype, size); - } - - private Token readToken_pipe_amp(int code) { // '|&' - int next = charAt(this.pos + 1); - if (next == code) - return this.finishOp(code == 124 ? TokenType.logicalOR : TokenType.logicalAND, 2); - if (next == 61) - return this.finishOp(TokenType.assign, 2); - return this.finishOp(code == 124 ? TokenType.bitwiseOR : TokenType.bitwiseAND, 1); - } - - private Token readToken_caret() { // '^' - int next = charAt(this.pos + 1); - if (next == 61) - return this.finishOp(TokenType.assign, 2); - return this.finishOp(TokenType.bitwiseXOR, 1); - } - - private Token readToken_plus_min(int code) { // '+-' - int next = charAt(this.pos + 1); - if (next == code) { - if (next == 45 && charAt(this.pos + 2) == 62 && - inputSubstring(this.lastTokEnd, this.pos).matches("(?s).*(?:" + lineBreak + ").*")) { - // A `-->` line comment - this.skipLineComment(3); - this.skipSpace(); - return this.nextToken(); - } - return this.finishOp(TokenType.incDec, 2); - } - if (next == 61) - return this.finishOp(TokenType.assign, 2); - return this.finishOp(TokenType.plusMin, 1); - } - - private Token readToken_lt_gt(int code) { // '<>' - int next = charAt(this.pos + 1); - int size = 1; - if (next == code) { - size = code == 62 && charAt(this.pos + 2) == 62 ? 3 : 2; - if (charAt(this.pos + size) == 61) - return this.finishOp(TokenType.assign, size + 1); - return this.finishOp(TokenType.bitShift, size); - } - if (next == 33 && code == 60 && charAt(this.pos + 2) == 45 && - charAt(this.pos + 3) == 45) { - if (this.inModule) - this.unexpected(); - // `` line comment + this.skipLineComment(3); + this.skipSpace(); + return this.nextToken(); + } + return this.finishOp(TokenType.incDec, 2); + } + if (next == 61) return this.finishOp(TokenType.assign, 2); + return this.finishOp(TokenType.plusMin, 1); + } + + private Token readToken_lt_gt(int code) { // '<>' + int next = charAt(this.pos + 1); + int size = 1; + if (next == code) { + size = code == 62 && charAt(this.pos + 2) == 62 ? 3 : 2; + if (charAt(this.pos + size) == 61) return this.finishOp(TokenType.assign, size + 1); + return this.finishOp(TokenType.bitShift, size); + } + if (next == 33 && code == 60 && charAt(this.pos + 2) == 45 && charAt(this.pos + 3) == 45) { + if (this.inModule) this.unexpected(); + // `0) { - res.append(readHexDigit()); - } - if (res.length() == 0) - return "0"; - return res.toString(); - } - - private String readDigits(boolean opt) { - StringBuilder res = new StringBuilder(); - for (char c=peekChar(true); c >= '0' && c <= '9'; nextChar(), c=peekChar(true)) - res.append(c); - if (res.length() == 0 && !opt) - this.error(Error.EXPECTED_DIGIT); - return res.toString(); - } - - private Double toNumber(String s) { - if (s.isEmpty()) - return 0.0; - return Double.valueOf(s); - } - - private String readIdentifier() { - StringBuilder res = new StringBuilder(); - for (char c=peekChar(true); - c != '\0' && Character.isJavaIdentifierPart(c); - nextChar(), c=peekChar(true)) - res.append(c); - if (res.length() == 0) - this.error(Error.EXPECTED_IDENTIFIER); - return res.toString(); - } - - private void expectRParen() { - if (!this.match(")")) - this.error(Error.EXPECTED_CLOSING_PAREN, this.pos-1); - } - - private void expectRBrace() { - if (!this.match("}")) - this.error(Error.EXPECTED_CLOSING_BRACE, this.pos-1); - } - - private void expectRAngle() { - if (!this.match(">")) - this.error(Error.EXPECTED_CLOSING_ANGLE, this.pos-1); - } - - private boolean lookahead(String... arguments) { - for (String prefix : arguments) { - if (prefix == null) { - if (atEOS()) - return true; - } else if (inputSubstring(pos, pos+prefix.length()).equals(prefix)) { - return true; - } - } - return false; - } - - private boolean match(String... arguments) { - for (String prefix : arguments) { - if (this.lookahead(prefix)) { - if (prefix == null) - prefix = ""; - this.pos += prefix.length(); - return true; - } - } - return false; - } - - private RegExpTerm parsePattern() { - RegExpTerm res = parseDisjunction(); - if (!this.atEOS()) - this.error(Error.EXPECTED_EOS); - return res; - } - - protected String inputSubstring(int start, int end) { - if (start >= src.length()) - return ""; - if (end > src.length()) - end = src.length(); - return src.substring(start, end); - } - - private T finishTerm(T term) { - SourceLocation loc = term.getLoc(); - Position end = pos(); - loc.setSource(inputSubstring(loc.getStart().getOffset(), end.getOffset())); - loc.setEnd(end); - return term; - } - - private RegExpTerm parseDisjunction() { - SourceLocation loc = new SourceLocation(pos()); - List disjuncts = new ArrayList<>(); - disjuncts.add(this.parseAlternative()); - while (this.match("|")) - disjuncts.add(this.parseAlternative()); - if (disjuncts.size() == 1) - return disjuncts.get(0); - return this.finishTerm(new Disjunction(loc, disjuncts)); - } - - private RegExpTerm parseAlternative() { - SourceLocation loc = new SourceLocation(pos()); - List elements = new ArrayList<>(); - while (!this.lookahead(null, "|", ")")) - elements.add(this.parseTerm()); - if (elements.size() == 1) - return elements.get(0); - return this.finishTerm(new Sequence(loc, elements)); - } - - private RegExpTerm parseTerm() { - SourceLocation loc = new SourceLocation(pos()); - - if (this.match("^")) - return this.finishTerm(new Caret(loc)); - - if (this.match("$")) - return this.finishTerm(new Dollar(loc)); - - if (this.match("\\b")) - return this.finishTerm(new WordBoundary(loc)); - - if (this.match("\\B")) - return this.finishTerm(new NonWordBoundary(loc)); - - if (this.match("(?=")) { - RegExpTerm dis = this.parseDisjunction(); - this.expectRParen(); - return this.finishTerm(new ZeroWidthPositiveLookahead(loc, dis)); - } - - if (this.match("(?!")) { - RegExpTerm dis = this.parseDisjunction(); - this.expectRParen(); - return this.finishTerm(new ZeroWidthNegativeLookahead(loc, dis)); - } - - if (this.match("(?<=")) { - RegExpTerm dis = this.parseDisjunction(); - this.expectRParen(); - return this.finishTerm(new ZeroWidthPositiveLookbehind(loc, dis)); - } - - if (this.match("(?")); - } - - if (this.match("p{", "P{")) { - String name = this.readIdentifier(); - if (this.match("=")) { - value = this.readIdentifier(); - raw = "\\p{" + name + "=" + value + "}"; - } else { - value = null; - raw = "\\p{" + name + "}"; - } - this.expectRBrace(); - return this.finishTerm(new UnicodePropertyEscape(loc, name, value, raw)); - } - - int startpos = this.pos-1; - char c = this.nextChar(); - - if (c >= '0' && c <= '9') { - raw = c + this.readDigits(true); - if (c == '0' || inCharClass) { - int base = c == '0' && raw.length() > 1 ? 8 : 10; - try { - codepoint = Long.parseLong(raw, base); - value = fromCodePoint((int) codepoint); - } catch (NumberFormatException nfe) { - codepoint = 0; - value = "\0"; - } - if (base == 8) { - this.error(Error.OCTAL_ESCAPE, startpos, this.pos); - return this.finishTerm(new OctalEscape(loc, value, (double)codepoint, "\\" + raw)); - } else { - return this.finishTerm(new DecimalEscape(loc, value, (double)codepoint, "\\" + raw)); - } - } else { - try { - codepoint = Long.parseLong(raw, 10); - } catch (NumberFormatException nfe) { - codepoint = 0; - } - BackReference br = this.finishTerm(new BackReference(loc, (double)codepoint, "\\" + raw)); - this.backrefs.add(br); - return br; - } - } - - String ctrltab = "f\fn\nr\rt\tv\u000b"; - int idx; - if ((idx=ctrltab.indexOf(c)) % 2 == 0) { - codepoint = ctrltab.charAt(idx+1); - value = String.valueOf((char)codepoint); - return this.finishTerm(new ControlEscape(loc, value, codepoint, "\\" + c)); - } - - if (c == 'c') { - c = this.nextChar(); - if (!(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z')) - this.error(Error.EXPECTED_CONTROL_LETTER, this.pos-1); - codepoint = c % 32; - value = String.valueOf((char)codepoint); - return this.finishTerm(new ControlLetter(loc, value, codepoint, "\\c" + c)); - } - - if ("dDsSwW".indexOf(c) >= 0) { - return this.finishTerm(new CharacterClassEscape(loc, String.valueOf(c), "\\" + c)); - } - - codepoint = c; - value = String.valueOf((char)codepoint); - return this.finishTerm(new IdentityEscape(loc, value, codepoint, "\\" + c)); - } - - private RegExpTerm parseCharacterClass() { - SourceLocation loc = new SourceLocation(pos()); - List elements = new ArrayList<>(); - - this.match("["); - boolean inverted = this.match("^"); - while (!this.match("]")) { - if (this.atEOS()) { - this.error(Error.EXPECTED_RBRACKET); - break; - } - elements.add(this.parseCharacterClassElement()); - } - return this.finishTerm(new CharacterClass(loc, elements, inverted)); - } - - private RegExpTerm parseCharacterClassElement() { - SourceLocation loc = new SourceLocation(pos()); - RegExpTerm atom = this.parseCharacterClassAtom(); - if (!this.lookahead("-]") && this.match("-")) - return this.finishTerm(new CharacterClassRange(loc, atom, this.parseCharacterClassAtom())); - return atom; - } - - private RegExpTerm parseCharacterClassAtom() { - SourceLocation loc = new SourceLocation(pos()); - char c = this.nextChar(); - if (c == '\\') { - if (this.match("b")) - return this.finishTerm(new ControlEscape(loc, "\b", 8, "\\b")); - return this.finishTerm(this.parseAtomEscape(loc, true)); - } - return this.finishTerm(new Constant(loc, String.valueOf(c))); - } + /** The result of a parse. */ + public static class Result { + /** The root of the parsed AST. */ + public final RegExpTerm ast; + + /** A list of errors encountered during parsing. */ + public final List errors; + + public Result(RegExpTerm ast, List errors) { + this.ast = ast; + this.errors = errors; + } + + public RegExpTerm getAST() { + return ast; + } + + public List getErrors() { + return errors; + } + } + + private String src; + private int pos; + private List errors; + private List backrefs; + private int maxbackref; + + /** Parse the given string as a regular expression. */ + public Result parse(String src) { + this.src = src; + this.pos = 0; + this.errors = new ArrayList<>(); + this.backrefs = new ArrayList<>(); + this.maxbackref = 0; + RegExpTerm root = parsePattern(); + for (BackReference backref : backrefs) + if (backref.getValue() > maxbackref) + errors.add(new Error(backref.getLoc(), Error.INVALID_BACKREF)); + return new Result(root, errors); + } + + private static String fromCodePoint(int codepoint) { + if (Character.isValidCodePoint(codepoint)) return new String(Character.toChars(codepoint)); + // replacement character + return "\ufffd"; + } + + private Position pos() { + return new Position(1, pos, pos); + } + + private void error(int code, int start, int end) { + Position startPos, endPos; + startPos = new Position(1, start, start); + endPos = new Position(1, end, end); + this.errors.add( + new Error(new SourceLocation(inputSubstring(start, end), startPos, endPos), code)); + } + + private void error(int code, int start) { + error(code, start, start + 1); + } + + private void error(int code) { + error(code, this.pos); + } + + private boolean atEOS() { + return pos >= src.length(); + } + + private char peekChar(boolean opt) { + if (this.atEOS()) { + if (!opt) this.error(Error.UNEXPECTED_EOS); + return '\0'; + } else { + return this.src.charAt(this.pos); + } + } + + private char nextChar() { + char c = peekChar(false); + if (this.pos < src.length()) ++this.pos; + return c; + } + + private String readHexDigit() { + char c = this.peekChar(false); + if (c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F') { + ++this.pos; + return String.valueOf(c); + } + if (c != '\0') this.error(Error.EXPECTED_HEX_DIGIT, this.pos); + return ""; + } + + private String readHexDigits(int n) { + StringBuilder res = new StringBuilder(); + while (n-- > 0) { + res.append(readHexDigit()); + } + if (res.length() == 0) return "0"; + return res.toString(); + } + + private String readDigits(boolean opt) { + StringBuilder res = new StringBuilder(); + for (char c = peekChar(true); c >= '0' && c <= '9'; nextChar(), c = peekChar(true)) + res.append(c); + if (res.length() == 0 && !opt) this.error(Error.EXPECTED_DIGIT); + return res.toString(); + } + + private Double toNumber(String s) { + if (s.isEmpty()) return 0.0; + return Double.valueOf(s); + } + + private String readIdentifier() { + StringBuilder res = new StringBuilder(); + for (char c = peekChar(true); + c != '\0' && Character.isJavaIdentifierPart(c); + nextChar(), c = peekChar(true)) res.append(c); + if (res.length() == 0) this.error(Error.EXPECTED_IDENTIFIER); + return res.toString(); + } + + private void expectRParen() { + if (!this.match(")")) this.error(Error.EXPECTED_CLOSING_PAREN, this.pos - 1); + } + + private void expectRBrace() { + if (!this.match("}")) this.error(Error.EXPECTED_CLOSING_BRACE, this.pos - 1); + } + + private void expectRAngle() { + if (!this.match(">")) this.error(Error.EXPECTED_CLOSING_ANGLE, this.pos - 1); + } + + private boolean lookahead(String... arguments) { + for (String prefix : arguments) { + if (prefix == null) { + if (atEOS()) return true; + } else if (inputSubstring(pos, pos + prefix.length()).equals(prefix)) { + return true; + } + } + return false; + } + + private boolean match(String... arguments) { + for (String prefix : arguments) { + if (this.lookahead(prefix)) { + if (prefix == null) prefix = ""; + this.pos += prefix.length(); + return true; + } + } + return false; + } + + private RegExpTerm parsePattern() { + RegExpTerm res = parseDisjunction(); + if (!this.atEOS()) this.error(Error.EXPECTED_EOS); + return res; + } + + protected String inputSubstring(int start, int end) { + if (start >= src.length()) return ""; + if (end > src.length()) end = src.length(); + return src.substring(start, end); + } + + private T finishTerm(T term) { + SourceLocation loc = term.getLoc(); + Position end = pos(); + loc.setSource(inputSubstring(loc.getStart().getOffset(), end.getOffset())); + loc.setEnd(end); + return term; + } + + private RegExpTerm parseDisjunction() { + SourceLocation loc = new SourceLocation(pos()); + List disjuncts = new ArrayList<>(); + disjuncts.add(this.parseAlternative()); + while (this.match("|")) disjuncts.add(this.parseAlternative()); + if (disjuncts.size() == 1) return disjuncts.get(0); + return this.finishTerm(new Disjunction(loc, disjuncts)); + } + + private RegExpTerm parseAlternative() { + SourceLocation loc = new SourceLocation(pos()); + List elements = new ArrayList<>(); + while (!this.lookahead(null, "|", ")")) elements.add(this.parseTerm()); + if (elements.size() == 1) return elements.get(0); + return this.finishTerm(new Sequence(loc, elements)); + } + + private RegExpTerm parseTerm() { + SourceLocation loc = new SourceLocation(pos()); + + if (this.match("^")) return this.finishTerm(new Caret(loc)); + + if (this.match("$")) return this.finishTerm(new Dollar(loc)); + + if (this.match("\\b")) return this.finishTerm(new WordBoundary(loc)); + + if (this.match("\\B")) return this.finishTerm(new NonWordBoundary(loc)); + + if (this.match("(?=")) { + RegExpTerm dis = this.parseDisjunction(); + this.expectRParen(); + return this.finishTerm(new ZeroWidthPositiveLookahead(loc, dis)); + } + + if (this.match("(?!")) { + RegExpTerm dis = this.parseDisjunction(); + this.expectRParen(); + return this.finishTerm(new ZeroWidthNegativeLookahead(loc, dis)); + } + + if (this.match("(?<=")) { + RegExpTerm dis = this.parseDisjunction(); + this.expectRParen(); + return this.finishTerm(new ZeroWidthPositiveLookbehind(loc, dis)); + } + + if (this.match("(?")); + } + + if (this.match("p{", "P{")) { + String name = this.readIdentifier(); + if (this.match("=")) { + value = this.readIdentifier(); + raw = "\\p{" + name + "=" + value + "}"; + } else { + value = null; + raw = "\\p{" + name + "}"; + } + this.expectRBrace(); + return this.finishTerm(new UnicodePropertyEscape(loc, name, value, raw)); + } + + int startpos = this.pos - 1; + char c = this.nextChar(); + + if (c >= '0' && c <= '9') { + raw = c + this.readDigits(true); + if (c == '0' || inCharClass) { + int base = c == '0' && raw.length() > 1 ? 8 : 10; + try { + codepoint = Long.parseLong(raw, base); + value = fromCodePoint((int) codepoint); + } catch (NumberFormatException nfe) { + codepoint = 0; + value = "\0"; + } + if (base == 8) { + this.error(Error.OCTAL_ESCAPE, startpos, this.pos); + return this.finishTerm(new OctalEscape(loc, value, (double) codepoint, "\\" + raw)); + } else { + return this.finishTerm(new DecimalEscape(loc, value, (double) codepoint, "\\" + raw)); + } + } else { + try { + codepoint = Long.parseLong(raw, 10); + } catch (NumberFormatException nfe) { + codepoint = 0; + } + BackReference br = this.finishTerm(new BackReference(loc, (double) codepoint, "\\" + raw)); + this.backrefs.add(br); + return br; + } + } + + String ctrltab = "f\fn\nr\rt\tv\u000b"; + int idx; + if ((idx = ctrltab.indexOf(c)) % 2 == 0) { + codepoint = ctrltab.charAt(idx + 1); + value = String.valueOf((char) codepoint); + return this.finishTerm(new ControlEscape(loc, value, codepoint, "\\" + c)); + } + + if (c == 'c') { + c = this.nextChar(); + if (!(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z')) + this.error(Error.EXPECTED_CONTROL_LETTER, this.pos - 1); + codepoint = c % 32; + value = String.valueOf((char) codepoint); + return this.finishTerm(new ControlLetter(loc, value, codepoint, "\\c" + c)); + } + + if ("dDsSwW".indexOf(c) >= 0) { + return this.finishTerm(new CharacterClassEscape(loc, String.valueOf(c), "\\" + c)); + } + + codepoint = c; + value = String.valueOf((char) codepoint); + return this.finishTerm(new IdentityEscape(loc, value, codepoint, "\\" + c)); + } + + private RegExpTerm parseCharacterClass() { + SourceLocation loc = new SourceLocation(pos()); + List elements = new ArrayList<>(); + + this.match("["); + boolean inverted = this.match("^"); + while (!this.match("]")) { + if (this.atEOS()) { + this.error(Error.EXPECTED_RBRACKET); + break; + } + elements.add(this.parseCharacterClassElement()); + } + return this.finishTerm(new CharacterClass(loc, elements, inverted)); + } + + private RegExpTerm parseCharacterClassElement() { + SourceLocation loc = new SourceLocation(pos()); + RegExpTerm atom = this.parseCharacterClassAtom(); + if (!this.lookahead("-]") && this.match("-")) + return this.finishTerm(new CharacterClassRange(loc, atom, this.parseCharacterClassAtom())); + return atom; + } + + private RegExpTerm parseCharacterClassAtom() { + SourceLocation loc = new SourceLocation(pos()); + char c = this.nextChar(); + if (c == '\\') { + if (this.match("b")) return this.finishTerm(new ControlEscape(loc, "\b", 8, "\\b")); + return this.finishTerm(this.parseAtomEscape(loc, true)); + } + return this.finishTerm(new Constant(loc, String.valueOf(c))); + } } diff --git a/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java b/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java index 1e68f5dd812d..8ce33fb4b465 100644 --- a/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java +++ b/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java @@ -1,13 +1,5 @@ package com.semmle.js.parser; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonNull; @@ -32,6 +24,7 @@ import com.semmle.js.ast.ConditionalExpression; import com.semmle.js.ast.ContinueStatement; import com.semmle.js.ast.DebuggerStatement; +import com.semmle.js.ast.DeclarationFlags; import com.semmle.js.ast.Decorator; import com.semmle.js.ast.DoWhileStatement; import com.semmle.js.ast.DynamicImport; @@ -63,7 +56,6 @@ import com.semmle.js.ast.LogicalExpression; import com.semmle.js.ast.MemberDefinition; import com.semmle.js.ast.MemberExpression; -import com.semmle.js.ast.DeclarationFlags; import com.semmle.js.ast.MetaProperty; import com.semmle.js.ast.MethodDefinition; import com.semmle.js.ast.MethodDefinition.Kind; @@ -149,2268 +141,2357 @@ import com.semmle.ts.ast.TypeofTypeExpr; import com.semmle.ts.ast.UnionTypeExpr; import com.semmle.util.collections.CollectionUtil; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** - * Utility class for converting a - * TypeScript AST node - * into a {@link Result}. + * Utility class for converting a TypeScript AST + * node into a {@link Result}. * - * TypeScript AST nodes that have no JavaScript equivalent are omitted. + *

      TypeScript AST nodes that have no JavaScript equivalent are omitted. */ public class TypeScriptASTConverter { - private String source; - private final JsonObject nodeFlags; - private final JsonObject syntaxKinds; - private final Map nodeFlagMap = new LinkedHashMap<>(); - private final Map syntaxKindMap = new LinkedHashMap<>(); - private int[] lineStarts; - - private int syntaxKindExtends; - - private final static Pattern LINE_TERMINATOR = Pattern.compile("\\n|\\r\\n|\\r|\\u2028|\\u2029"); - private static final String WHITESPACE_CHAR = "(?:\\s|//.*|/\\*(?:[^*]|\\*(?!/))*\\*/)"; - private static final Pattern WHITESPACE = Pattern.compile("^" + WHITESPACE_CHAR + "*"); - private static final Pattern EXPORT_DECL_START = Pattern.compile("^export" + - "(" + WHITESPACE_CHAR + "+default)?" + - WHITESPACE_CHAR + "+"); - private static final Pattern TYPEOF_START = Pattern.compile("^typeof" + WHITESPACE_CHAR + "+"); - private static final Pattern WHITESPACE_END_PAREN = Pattern.compile("^" + WHITESPACE_CHAR + "*\\)"); - - TypeScriptASTConverter(JsonObject nodeFlags, JsonObject syntaxKinds) { - this.nodeFlags = nodeFlags; - this.syntaxKinds = syntaxKinds; - makeEnumIdMap(nodeFlags, nodeFlagMap); - makeEnumIdMap(syntaxKinds, syntaxKindMap); - this.syntaxKindExtends = getSyntaxKind("ExtendsKeyword"); - } - - /** - * Builds a mapping from ID to name given a TypeScript enum object. - */ - private void makeEnumIdMap(JsonObject enumObject, Map idToName) { - for (Map.Entry entry : enumObject.entrySet()) { - JsonPrimitive prim = entry.getValue().getAsJsonPrimitive(); - if (prim.isNumber() && !idToName.containsKey(prim.getAsInt())) { - idToName.put(prim.getAsInt(), entry.getKey()); - } - } - } - - /** - * Convert the given TypeScript AST (which was parsed from {@code source}) - * into a parser {@link Result}. - */ - public Result convertAST(JsonObject ast, String source) { - this.lineStarts = toIntArray(ast.getAsJsonArray("$lineStarts")); - - List errors = new ArrayList(); - - // process parse diagnostics (i.e., syntax errors) reported by the TypeScript compiler - JsonArray parseDiagnostics = ast.get("parseDiagnostics").getAsJsonArray(); - if (parseDiagnostics.size() > 0) { - for (JsonElement elt : parseDiagnostics) { - JsonObject parseDiagnostic = elt.getAsJsonObject(); - String message = parseDiagnostic.get("messageText").getAsString(); - Position pos = getPosition(parseDiagnostic.get("$pos")); - errors.add(new ParseError(message, pos.getLine(), pos.getColumn(), pos.getOffset())); - } - return new Result(source, null, new ArrayList<>(), new ArrayList<>(), errors); - } - - this.source = source; - - List tokens = new ArrayList<>(); - List comments = new ArrayList<>(); - extractTokensAndComments(ast, tokens, comments); - Node converted; - try { - converted = convertNode(ast); - } catch (ParseError e) { - converted = null; - errors.add(e); - } - return new Result(source, converted, tokens, comments, errors); - } - - /** - * Converts a JSON array to an int array. - * The array is assumed to only contain integers. - */ - private static int[] toIntArray(JsonArray array) { - int[] result = new int[array.size()]; - for (int i = 0; i < array.size(); ++i) { - result[i] = array.get(i).getAsInt(); - } - return result; - } - - private int getLineFromPos(int pos) { - int low = 0, high = this.lineStarts.length - 1; - while (low < high) { - int mid = high - ((high - low) >> 1); // Get middle, rounding up. - int startOfLine = lineStarts[mid]; - if (startOfLine <= pos) { - low = mid; - } else { - high = mid - 1; - } - } - return low; - } - - private int getColumnFromLinePos(int line, int pos) { - return pos - lineStarts[line]; - } - - /** - * Extract tokens and comments from the given TypeScript AST. - */ - private void extractTokensAndComments(JsonObject ast, List tokens, List comments) { - for (JsonElement elt : ast.get("$tokens").getAsJsonArray()) { - JsonObject token = elt.getAsJsonObject(); - String text = token.get("text").getAsString(); - Position start = getPosition(token.get("tokenPos")); - Position end = advance(start, text); - SourceLocation loc = new SourceLocation(text, start, end); - String kind = getKind(token); - switch (kind) { - case "EndOfFileToken": - tokens.add(new Token(loc, Token.Type.EOF)); - break; - case "SingleLineCommentTrivia": - case "MultiLineCommentTrivia": - String cookedText; - if (text.startsWith("//")) - cookedText = text.substring(2); - else - cookedText = text.substring(2, text.length()-2); - comments.add(new Comment(loc, cookedText)); - break; - case "TemplateHead": - case "TemplateMiddle": - case "TemplateTail": - case "NoSubstitutionTemplateLiteral": - tokens.add(new Token(loc, Token.Type.STRING)); - break; - case "Identifier": - tokens.add(new Token(loc, Token.Type.NAME)); - break; - case "NumericLiteral": - tokens.add(new Token(loc, Token.Type.NUM)); - break; - case "StringLiteral": - tokens.add(new Token(loc, Token.Type.STRING)); - break; - case "RegularExpressionLiteral": - tokens.add(new Token(loc, Token.Type.REGEXP)); - break; - default: - Token.Type tp; - if (kind.endsWith("Token")) { - tp = Token.Type.PUNCTUATOR; - } else if (kind.endsWith("Keyword")) { - if (text.equals("null")) - tp = Token.Type.NULL; - else if (text.equals("true")) - tp = Token.Type.TRUE; - else if (text.equals("false")) - tp = Token.Type.FALSE; - else - tp = Token.Type.KEYWORD; - } else { - continue; - } - tokens.add(new Token(loc, tp)); - } - } - } - - /** - * Convert the given TypeScript node and its children into a JavaScript {@link Node}. - */ - private Node convertNode(JsonObject node) throws ParseError { - return convertNode(node, null); - } - - /** - * Convert the given TypeScript node and its children into a JavaScript - * {@link Node}. If the TypesScript node has no explicit {@code kind}, it is - * assumed to be {@code defaultKind}. - */ - private Node convertNode(JsonObject node, String defaultKind) throws ParseError { - Node astNode = convertNodeUntyped(node, defaultKind); - attachStaticType(astNode, node); - return astNode; - } - - /** - * Helper method for `convertNode` that does everything except attaching type - * information. - */ - private Node convertNodeUntyped(JsonObject node, String defaultKind) throws ParseError { - String kind = getKind(node); - if (kind == null) - kind = defaultKind; - if (kind == null) - kind = "Identifier"; - SourceLocation loc = getSourceLocation(node); - switch (kind) { - case "AnyKeyword": - return convertKeywordTypeExpr(node, loc, "any"); - case "ArrayBindingPattern": - return convertArrayBindingPattern(node, loc); - case "ArrayLiteralExpression": - return convertArrayLiteralExpression(node, loc); - case "ArrayType": - return convertArrayType(node, loc); - case "ArrowFunction": - return convertArrowFunction(node, loc); - case "AsExpression": - return convertAsExpression(node, loc); - case "AwaitExpression": - return convertAwaitExpression(node, loc); - case "BigIntKeyword": - return convertKeywordTypeExpr(node, loc, "bigint"); - case "BigIntLiteral": - return convertBigIntLiteral(node, loc); - case "BinaryExpression": - return convertBinaryExpression(node, loc); - case "Block": - return convertBlock(node, loc); - case "BooleanKeyword": - return convertKeywordTypeExpr(node, loc, "boolean"); - case "BreakStatement": - return convertBreakStatement(node, loc); - case "CallExpression": - return convertCallExpression(node, loc); - case "CallSignature": - return convertCallSignature(node, loc); - case "CaseClause": - return convertCaseClause(node, loc); - case "CatchClause": - return convertCatchClause(node, loc); - case "ClassDeclaration": - case "ClassExpression": - return convertClass(node, kind, loc); - case "CommaListExpression": - return convertCommaListExpression(node, loc); - case "ComputedPropertyName": - return convertComputedPropertyName(node); - case "ConditionalExpression": - return convertConditionalExpression(node, loc); - case "ConditionalType": - return convertConditionalType(node, loc); - case "Constructor": - return convertConstructor(node, loc); - case "ConstructSignature": - return convertConstructSignature(node, loc); - case "ConstructorType": - return convertConstructorType(node, loc); - case "ContinueStatement": - return convertContinueStatement(node, loc); - case "DebuggerStatement": - return convertDebuggerStatement(loc); - case "Decorator": - return convertDecorator(node, loc); - case "DefaultClause": - return convertCaseClause(node, loc); - case "DeleteExpression": - return convertDeleteExpression(node, loc); - case "DoStatement": - return convertDoStatement(node, loc); - case "ElementAccessExpression": - return convertElementAccessExpression(node, loc); - case "EmptyStatement": - return convertEmptyStatement(loc); - case "EnumDeclaration": - return convertEnumDeclaration(node, loc); - case "EnumMember": - return convertEnumMember(node, loc); - case "ExportAssignment": - return convertExportAssignment(node, loc); - case "ExportDeclaration": - return convertExportDeclaration(node, loc); - case "ExportSpecifier": - return convertExportSpecifier(node, loc); - case "ExpressionStatement": - return convertExpressionStatement(node, loc); - case "ExpressionWithTypeArguments": - return convertExpressionWithTypeArguments(node, loc); - case "ExternalModuleReference": - return convertExternalModuleReference(node, loc); - case "FalseKeyword": - return convertFalseKeyword(loc); - case "NeverKeyword": - return convertKeywordTypeExpr(node, loc, "never"); - case "NumberKeyword": - return convertKeywordTypeExpr(node, loc, "number"); - case "NumericLiteral": - return convertNumericLiteral(node, loc); - case "ForStatement": - return convertForStatement(node, loc); - case "ForInStatement": - return convertForInStatement(node, loc); - case "ForOfStatement": - return convertForOfStatement(node, loc); - case "FunctionDeclaration": - return convertFunctionDeclaration(node, loc); - case "FunctionExpression": - return convertFunctionExpression(node, loc); - case "FunctionType": - return convertFunctionType(node, loc); - case "Identifier": - return convertIdentifier(node, loc); - case "IfStatement": - return convertIfStatement(node, loc); - case "ImportClause": - return convertImportClause(node, loc); - case "ImportDeclaration": - return convertImportDeclaration(node, loc); - case "ImportEqualsDeclaration": - return convertImportEqualsDeclaration(node, loc); - case "ImportKeyword": - return convertImportKeyword(loc); - case "ImportSpecifier": - return convertImportSpecifier(node, loc); - case "ImportType": - return convertImportType(node, loc); - case "IndexSignature": - return convertIndexSignature(node, loc); - case "IndexedAccessType": - return convertIndexedAccessType(node, loc); - case "InferType": - return convertInferType(node, loc); - case "InterfaceDeclaration": - return convertInterfaceDeclaration(node, loc); - case "IntersectionType": - return convertIntersectionType(node, loc); - case "JsxAttribute": - return convertJsxAttribute(node, loc); - case "JsxClosingElement": - return convertJsxClosingElement(node, loc); - case "JsxElement": - return convertJsxElement(node, loc); - case "JsxExpression": - return convertJsxExpression(node, loc); - case "JsxFragment": - return convertJsxFragment(node, loc); - case "JsxOpeningElement": - return convertJsxOpeningElement(node, loc); - case "JsxOpeningFragment": - return convertJsxOpeningFragment(node, loc); - case "JsxSelfClosingElement": - return convertJsxSelfClosingElement(node, loc); - case "JsxClosingFragment": - return convertJsxClosingFragment(node, loc); - case "JsxSpreadAttribute": - return convertJsxSpreadAttribute(node, loc); - case "JsxText": - case "JsxTextAllWhiteSpaces": - return convertJsxText(node, loc); - case "LabeledStatement": - return convertLabeledStatement(node, loc); - case "LiteralType": - return convertLiteralType(node, loc); - case "MappedType": - return convertMappedType(node, loc); - case "MetaProperty": - return convertMetaProperty(node, loc); - case "GetAccessor": - case "SetAccessor": - case "MethodDeclaration": - case "MethodSignature": - return convertMethodDeclaration(node, kind, loc); - case "ModuleDeclaration": - case "NamespaceDeclaration": - return convertNamespaceDeclaration(node, loc); - case "ModuleBlock": - return convertModuleBlock(node, loc); - case "NamespaceExportDeclaration": - return convertNamespaceExportDeclaration(node, loc); - case "NamespaceImport": - return convertNamespaceImport(node, loc); - case "NewExpression": - return convertNewExpression(node, loc); - case "NonNullExpression": - return convertNonNullExpression(node, loc); - case "NoSubstitutionTemplateLiteral": - return convertNoSubstitutionTemplateLiteral(node, loc); - case "NullKeyword": - return convertNullKeyword(loc); - case "ObjectBindingPattern": - return convertObjectBindingPattern(node, loc); - case "ObjectKeyword": - return convertKeywordTypeExpr(node, loc, "object"); - case "ObjectLiteralExpression": - return convertObjectLiteralExpression(node, loc); - case "OmittedExpression": - return convertOmittedExpression(); - case "OptionalType": - return convertOptionalType(node, loc); - case "Parameter": - return convertParameter(node, loc); - case "ParenthesizedExpression": - return convertParenthesizedExpression(node, loc); - case "ParenthesizedType": - return convertParenthesizedType(node, loc); - case "PostfixUnaryExpression": - return convertPostfixUnaryExpression(node, loc); - case "PrefixUnaryExpression": - return convertPrefixUnaryExpression(node, loc); - case "PropertyAccessExpression": - return convertPropertyAccessExpression(node, loc); - case "PropertyAssignment": - return convertPropertyAssignment(node, loc); - case "PropertyDeclaration": - case "PropertySignature": - return convertPropertyDeclaration(node, kind, loc); - case "RegularExpressionLiteral": - return convertRegularExpressionLiteral(loc); - case "RestType": - return convertRestType(node, loc); - case "QualifiedName": - return convertQualifiedName(node, loc); - case "ReturnStatement": - return convertReturnStatement(node, loc); - case "SemicolonClassElement": - return convertSemicolonClassElement(); - case "SourceFile": - return convertSourceFile(node, loc); - case "ShorthandPropertyAssignment": - return convertShorthandPropertyAssignment(node, loc); - case "SpreadAssignment": - case "SpreadElement": - case "SpreadElementExpression": - return convertSpreadElement(node, loc); - case "StringKeyword": - return convertKeywordTypeExpr(node, loc, "string"); - case "StringLiteral": - return convertStringLiteral(node, loc); - case "SuperKeyword": - return convertSuperKeyword(loc); - case "SwitchStatement": - return convertSwitchStatement(node, loc); - case "SymbolKeyword": - return convertKeywordTypeExpr(node, loc, "symbol"); - case "TaggedTemplateExpression": - return convertTaggedTemplateExpression(node, loc); - case "TemplateExpression": - return convertTemplateExpression(node, loc); - case "TemplateHead": - case "TemplateMiddle": - case "TemplateTail": - return convertTemplateElement(node, kind, loc); - case "ThisKeyword": - return convertThisKeyword(loc); - case "ThisType": - return convertKeywordTypeExpr(node, loc, "this"); - case "ThrowStatement": - return convertThrowStatement(node, loc); - case "TrueKeyword": - return convertTrueKeyword(loc); - case "TryStatement": - return convertTryStatement(node, loc); - case "TupleType": - return convertTupleType(node, loc); - case "TypeAliasDeclaration": - return convertTypeAliasDeclaration(node, loc); - case "TypeAssertionExpression": - return convertTypeAssertionExpression(node, loc); - case "TypeLiteral": - return convertTypeLiteral(node, loc); - case "TypeOfExpression": - return convertTypeOfExpression(node, loc); - case "TypeOperator": - return convertTypeOperator(node, loc); - case "TypeParameter": - return convertTypeParameter(node, loc); - case "TypePredicate": - return convertTypePredicate(node, loc); - case "TypeReference": - return convertTypeReference(node, loc); - case "TypeQuery": - return convertTypeQuery(node, loc); - case "UndefinedKeyword": - return convertKeywordTypeExpr(node, loc, "undefined"); - case "UnionType": - return convertUnionType(node, loc); - case "UnknownKeyword": - return convertKeywordTypeExpr(node, loc, "unknown"); - case "VariableDeclaration": - return convertVariableDeclaration(node, loc); - case "VariableDeclarationList": - return convertVariableDeclarationList(node, loc); - case "VariableStatement": - return convertVariableStatement(node, loc); - case "VoidExpression": - return convertVoidExpression(node, loc); - case "VoidKeyword": - return convertKeywordTypeExpr(node, loc, "void"); - case "WhileStatement": - return convertWhileStatement(node, loc); - case "WithStatement": - return convertWithStatement(node, loc); - case "YieldExpression": - return convertYieldExpression(node, loc); - default: - throw new ParseError("Unsupported TypeScript syntax " + kind, getSourceLocation(node).getStart()); - } - } - - /** - * Attaches type information from the JSON object to the given AST node, if - * applicable. This is called from {@link #convertNode}. - */ - private void attachStaticType(Node astNode, JsonObject json) { - if (astNode instanceof ITypedAstNode && json.has("$type")) { - ITypedAstNode typedAstNode = (ITypedAstNode) astNode; - int typeId = json.get("$type").getAsInt(); - typedAstNode.setStaticTypeId(typeId); - } - } - - /** - * Attaches a TypeScript compiler symbol to the given node, if any was provided. - */ - private void attachSymbolInformation(INodeWithSymbol node, JsonObject json) { - if (json.has("$symbol")) { - int symbol = json.get("$symbol").getAsInt(); - node.setSymbol(symbol); - } - } - - /** - * Attaches call signatures and related symbol information to a call site. - */ - private void attachResolvedSignature(InvokeExpression node, JsonObject json) { - if (json.has("$resolvedSignature")) { - int id = json.get("$resolvedSignature").getAsInt(); - node.setResolvedSignatureId(id); - } - if (json.has("$overloadIndex")) { - int id = json.get("$overloadIndex").getAsInt(); - node.setOverloadIndex(id); - } - attachSymbolInformation(node, json); - } - - /** - * Convert the given array of TypeScript AST nodes into a list of JavaScript AST nodes, - * skipping any {@code null} elements. - */ - private List convertNodes(Iterable nodes) throws ParseError { - return convertNodes(nodes, true); - } - - /** - * Convert the given array of TypeScript AST nodes into a list of JavaScript AST nodes, - * where {@code skipNull} indicates whether {@code null} elements should be skipped or not. - */ - @SuppressWarnings("unchecked") - private List convertNodes(Iterable nodes, - boolean skipNull) throws ParseError { - List res = new ArrayList(); - for (JsonElement elt : nodes) { - T converted = (T)convertNode(elt.getAsJsonObject()); - if (!skipNull || converted != null) - res.add(converted); - } - return res; - } - - /** - * Converts the given child to an AST node of the given type or null. - * A ParseError is thrown if a different type of node was found. - *

      - * This is used to detect syntax errors that are not reported as syntax errors - * by the TypeScript parser. Usually they are reported as errors in a later - * compiler stage, which the extractor does not run. - *

      - * Returns null if the child is absent. - */ - @SuppressWarnings("unchecked") - private T tryConvertChild(JsonObject node, String prop, Class expectedType) throws ParseError { - Node child = convertChild(node, prop); - if (child == null || expectedType.isInstance(child)) { - return (T) child; - } else { - throw new ParseError("Unsupported TypeScript syntax", getSourceLocation(node).getStart()); - } - } - - /** - * Convert the child node named {@code prop} of AST node {@code node}. - */ - private T convertChild(JsonObject node, String prop) throws ParseError { - return convertChild(node, prop, null); - } - - /** - * Convert the child node named {@code prop} of AST node {@code node}, with - * {@code kind} as its default kind. - */ - @SuppressWarnings("unchecked") - private T convertChild(JsonObject node, String prop, String kind) throws ParseError { - JsonElement child = node.get(prop); - if (child == null) - return null; - return (T)convertNode(child.getAsJsonObject(), kind); - } - - /** - * Convert the child nodes named {@code prop} of AST node {@code node}. - */ - private List convertChildren(JsonObject node, String prop) throws ParseError { - return convertChildren(node, prop, true); - } - - /** - * Like convertChildren but returns an empty list if the property is missing. - */ - private List convertChildrenNotNull(JsonObject node, String prop) throws ParseError { - List nodes = convertChildren(node, prop, true); - if (nodes == null) { - return Collections.emptyList(); - } - return nodes; - } - - /** - * Convert the child nodes named {@code prop} of AST node {@code node}, where - * {@code skipNull} indicates whether or not to skip null children. - */ - private List convertChildren(JsonObject node, String prop, - boolean skipNull) throws ParseError { - JsonElement child = node.get(prop); - if (child == null) - return null; - return convertNodes(child.getAsJsonArray(), skipNull); - } - - /* Converter methods for the individual TypeScript AST node types. */ - - private Node convertArrayBindingPattern(JsonObject array, SourceLocation loc) throws ParseError { - List elements = new ArrayList<>(); - for (JsonElement elt : array.get("elements").getAsJsonArray()) { - JsonObject element = (JsonObject) elt; - SourceLocation eltLoc = getSourceLocation(element); - Expression convertedElt = convertChild(element, "name"); - if (hasChild(element, "initializer")) - convertedElt = new AssignmentPattern(eltLoc, "=", convertedElt, convertChild(element, "initializer")); - else if (hasChild(element, "dotDotDotToken")) - convertedElt = new RestElement(eltLoc, convertedElt); - elements.add(convertedElt); - } - return new ArrayPattern(loc, elements); - } - - private Node convertArrayLiteralExpression(JsonObject node, SourceLocation loc) throws ParseError { - return new ArrayExpression(loc, convertChildren(node, "elements", false)); - } - - private Node convertArrayType(JsonObject node, SourceLocation loc) throws ParseError { - return new ArrayTypeExpr(loc, convertChildAsType(node, "elementType")); - } - - private Node convertArrowFunction(JsonObject node, SourceLocation loc) throws ParseError { - return new ArrowFunctionExpression(loc, convertParameters(node), convertChild(node, "body"), false, - hasModifier(node, "AsyncKeyword"), convertChildrenNotNull(node, "typeParameters"), convertParameterTypes(node), - convertChildAsType(node, "type")); - } - - private Node convertAsExpression(JsonObject node, SourceLocation loc) throws ParseError { - return new TypeAssertion(loc, convertChild(node, "expression"), convertChildAsType(node, "type"), true); - } - - private Node convertAwaitExpression(JsonObject node, SourceLocation loc) throws ParseError { - return new AwaitExpression(loc, convertChild(node, "expression")); - } - - private Node convertBigIntLiteral(JsonObject node, SourceLocation loc) throws ParseError { - String text = node.get("text").getAsString(); - String value = text.substring(0, text.length() - 1); // Remove the 'n' suffix. - return new Literal(loc, TokenType.bigint, value); - } - - private Node convertBinaryExpression(JsonObject node, SourceLocation loc) throws ParseError { - Expression left = convertChild(node, "left"); - Expression right = convertChild(node, "right"); - JsonObject operatorToken = node.get("operatorToken").getAsJsonObject(); - String operator = getSourceLocation(operatorToken).getSource(); - switch (operator) { - case ",": - List expressions = new ArrayList(); - if (left instanceof SequenceExpression) - expressions.addAll(((SequenceExpression) left).getExpressions()); - else - expressions.add(left); - if (right instanceof SequenceExpression) - expressions.addAll(((SequenceExpression) right).getExpressions()); - else - expressions.add(right); - return new SequenceExpression(loc, expressions); - - case "||": - case "&&": - return new LogicalExpression(loc, operator, left, right); - - case "=": - left = convertLValue(left); // For plain assignments, the lhs can be a destructuring pattern. - return new AssignmentExpression(loc, operator, left, right); - - case "+=": - case "-=": - case "*=": - case "**=": - case "/=": - case "%=": - case "^=": - case "&=": - case "|=": - case ">>=": - case "<<=": - case ">>>=": - return new AssignmentExpression(loc, operator, convertLValue(left), right); - - default: - return new BinaryExpression(loc, operator, left, right); - } - } - - private Node convertBlock(JsonObject node, SourceLocation loc) throws ParseError { - return new BlockStatement(loc, convertChildren(node, "statements")); - } - - private Node convertBreakStatement(JsonObject node, SourceLocation loc) throws ParseError { - return new BreakStatement(loc, convertChild(node, "label")); - } - - private Node convertCallExpression(JsonObject node, SourceLocation loc) throws ParseError { - List arguments = convertChildren(node, "arguments"); - if (arguments.size() == 1 && hasKind(node.get("expression"), "ImportKeyword")) { - return new DynamicImport(loc, arguments.get(0)); - } - Expression callee = convertChild(node, "expression"); - List typeArguments = convertChildrenAsTypes(node, "typeArguments"); - CallExpression call = new CallExpression(loc, callee, typeArguments, arguments, false, false); - attachResolvedSignature(call, node); - return call; - } - - private MethodDefinition convertCallSignature(JsonObject node, SourceLocation loc) throws ParseError { - FunctionExpression function = convertImplicitFunction(node, loc); - int flags = getMemberModifierKeywords(node) | DeclarationFlags.abstract_; - return new MethodDefinition(loc, flags, Kind.FUNCTION_CALL_SIGNATURE, null, function); - } - - private Node convertCaseClause(JsonObject node, SourceLocation loc) throws ParseError { - return convertDefaultClause(node, loc); - } - - private Node convertCatchClause(JsonObject node, SourceLocation loc) throws ParseError { - IPattern pattern = null; - JsonElement variableDecl = node.get("variableDeclaration"); - if (variableDecl != null) - pattern = convertChild(variableDecl.getAsJsonObject(), "name"); - return new CatchClause(loc, pattern, null, convertChild(node, "block")); - } - - private List convertSuperInterfaceClause(JsonArray supers) throws ParseError { - List result = new ArrayList<>(); - for (JsonElement elt : supers) { - JsonObject superType = elt.getAsJsonObject(); - ITypeExpression objectType = convertChildAsType(superType, "expression"); - if (objectType == null) - continue; - List typeArguments = convertChildrenAsTypes(superType, "typeArguments"); - if (typeArguments.isEmpty()) { - result.add(objectType); - } else { - result.add(new GenericTypeExpr(getSourceLocation(superType), objectType, typeArguments)); - } - } - return result; - } - - private Node convertClass(JsonObject node, String kind, SourceLocation loc) throws ParseError { - Identifier id = convertChild(node, "name"); - List typeParameters = convertChildrenNotNull(node, "typeParameters"); - Expression superClass = null; - List superInterfaces = null; - int afterHead = id == null ? loc.getStart().getOffset() + 5 : id.getLoc().getEnd().getOffset(); - for (JsonElement elt : getChildIterable(node, "heritageClauses")) { - JsonObject heritageClause = elt.getAsJsonObject(); - JsonArray supers = heritageClause.get("types").getAsJsonArray(); - if (heritageClause.get("token").getAsInt() == syntaxKindExtends) { - if (supers.size() > 0) { - superClass = (Expression) convertNode(supers.get(0).getAsJsonObject()); - } - } else { - superInterfaces = convertSuperInterfaceClause(supers); - } - afterHead = heritageClause.get("$end").getAsInt(); - } - if (superInterfaces == null) { - superInterfaces = new ArrayList<>(); - } - String skip = source.substring(loc.getStart().getOffset(), afterHead) + matchWhitespace(afterHead); - SourceLocation bodyLoc = new SourceLocation(loc.getSource(), loc.getStart(), loc.getEnd()); - advance(bodyLoc, skip); - ClassBody body = new ClassBody(bodyLoc, convertChildren(node, "members")); - if ("ClassExpression".equals(kind)) { - ClassExpression classExpr = new ClassExpression(loc, id, typeParameters, superClass, superInterfaces, body); - attachSymbolInformation(classExpr.getClassDef(), node); - return classExpr; - } - boolean hasDeclareKeyword = hasModifier(node, "DeclareKeyword"); - boolean hasAbstractKeyword = hasModifier(node, "AbstractKeyword"); - ClassDeclaration classDecl = new ClassDeclaration(loc, id, typeParameters, superClass, superInterfaces, body, hasDeclareKeyword, - hasAbstractKeyword); - attachSymbolInformation(classDecl.getClassDef(), node); - if (node.has("decorators")) { - classDecl.addDecorators(convertChildren(node, "decorators")); - advanceUntilAfter(loc, classDecl.getDecorators()); - } - return fixExports(loc, classDecl); - } - - private Node convertCommaListExpression(JsonObject node, - SourceLocation loc) throws ParseError { - return new SequenceExpression(loc, convertChildren(node, "elements")); - } - - private Node convertComputedPropertyName(JsonObject node) throws ParseError { - return convertChild(node, "expression"); - } - - private Node convertConditionalExpression(JsonObject node, - SourceLocation loc) throws ParseError { - return new ConditionalExpression(loc, convertChild(node, "condition"), convertChild(node, "whenTrue"), convertChild(node, "whenFalse")); - } - - private Node convertConditionalType(JsonObject node, SourceLocation loc) throws ParseError { - return new ConditionalTypeExpr(loc, convertChild(node, "checkType"), convertChild(node, "extendsType"), - convertChild(node, "trueType"), convertChild(node, "falseType")); - } - - private SourceLocation getSourceRange(Position from, Position to) { - return new SourceLocation(source.substring(from.getOffset(), to.getOffset()), from, to); - } - - private DecoratorList makeDecoratorList(JsonElement decorators) throws ParseError { - if (!(decorators instanceof JsonArray)) return null; - JsonArray array = decorators.getAsJsonArray(); - SourceLocation firstLoc = null, lastLoc = null; - List list = new ArrayList<>(); - for (JsonElement decoratorElm : array) { - JsonObject decorator = decoratorElm.getAsJsonObject(); - if (hasKind(decorator, "Decorator")) { - SourceLocation location = getSourceLocation(decorator); - list.add(convertDecorator(decorator, location)); - if (firstLoc == null) { - firstLoc = location; - } - lastLoc = location; - } - } - if (firstLoc == null) - return null; - return new DecoratorList(getSourceRange(firstLoc.getStart(), lastLoc.getEnd()), list); - } - - private List convertParameterDecorators(JsonObject function) throws ParseError { - List decoratorLists = new ArrayList<>(); - for (JsonElement parameter : getProperParameters(function)) { - decoratorLists.add(makeDecoratorList(parameter.getAsJsonObject().get("decorators"))); - } - return decoratorLists; - } - - private Node convertConstructor(JsonObject node, SourceLocation loc) throws ParseError { - int flags = getMemberModifierKeywords(node); - boolean isComputed = hasComputedName(node); - boolean isStatic = DeclarationFlags.isStatic(flags); - if (isComputed) { - flags |= DeclarationFlags.computed; - } - // for some reason, the TypeScript compiler treats static methods named "constructor" - // and methods with computed name "constructor" as constructors, even though they aren't - MethodDefinition.Kind methodKind = isStatic || isComputed ? Kind.METHOD : Kind.CONSTRUCTOR; - Expression key; - if (isComputed) - key = convertChild((JsonObject) node.get("name"), "expression"); - else - key = new Identifier(loc, "constructor"); - List params = convertParameters(node); - List paramTypes = convertParameterTypes(node); - List paramDecorators = convertParameterDecorators(node); - FunctionExpression value = new FunctionExpression(loc, null, params, convertChild(node, "body"), false, false, - Collections.emptyList(), paramTypes, paramDecorators, null, null); - attachSymbolInformation(value, node); - List parameterFields = convertParameterFields(node); - return new MethodDefinition(loc, flags, methodKind, key, value, parameterFields); - } - - private MethodDefinition convertConstructSignature(JsonObject node, SourceLocation loc) throws ParseError { - FunctionExpression function = convertImplicitFunction(node, loc); - int flags = getMemberModifierKeywords(node) | DeclarationFlags.abstract_; - return new MethodDefinition(loc, flags, Kind.CONSTRUCTOR_CALL_SIGNATURE, null, function); - } - - private Node convertConstructorType(JsonObject node, SourceLocation loc) throws ParseError { - return new FunctionTypeExpr(loc, convertImplicitFunction(node, loc), true); - } - - private Node convertContinueStatement(JsonObject node, SourceLocation loc) throws ParseError { - return new ContinueStatement(loc, convertChild(node, "label")); - } - - private Node convertDebuggerStatement(SourceLocation loc) { - return new DebuggerStatement(loc); - } - - private Decorator convertDecorator(JsonObject node, SourceLocation loc) throws ParseError { - return new Decorator(loc, convertChild(node, "expression")); - } - - private Node convertDefaultClause(JsonObject node, SourceLocation loc) throws ParseError { - return new SwitchCase(loc, convertChild(node, "expression"), convertChildren(node, "statements")); - } - - private Node convertDeleteExpression(JsonObject node, SourceLocation loc) throws ParseError { - return new UnaryExpression(loc, "delete", convertChild(node, "expression"), true); - } - - private Node convertDoStatement(JsonObject node, SourceLocation loc) throws ParseError { - return new DoWhileStatement(loc, convertChild(node, "expression"), convertChild(node, "statement")); - } - - private Node convertElementAccessExpression(JsonObject node, - SourceLocation loc) throws ParseError { - Expression object = convertChild(node, "expression"); - Expression property = convertChild(node, "argumentExpression"); - return new MemberExpression(loc, object, property, true, false, false); - } - - private Node convertEmptyStatement(SourceLocation loc) { - return new EmptyStatement(loc); - } - - private Node convertEnumDeclaration(JsonObject node, SourceLocation loc) throws ParseError { - EnumDeclaration enumDeclaration = new EnumDeclaration(loc, hasModifier(node, "ConstKeyword"), hasModifier(node, "DeclareKeyword"), - convertChildrenNotNull(node, "decorators"), convertChild(node, "name"), convertChildren(node, "members")); - attachSymbolInformation(enumDeclaration, node); - advanceUntilAfter(loc, enumDeclaration.getDecorators()); - return fixExports(loc, enumDeclaration); - } - - /** - * Converts a TypeScript Identifier or StringLiteral node to an Identifier AST - * node, or {@code null} if the given node is not of the expected kind. - */ - private Identifier convertNodeAsIdentifier(JsonObject node) throws ParseError { - SourceLocation loc = getSourceLocation(node); - if (isIdentifier(node)) { - return convertIdentifier(node, loc); - } else if (hasKind(node, "StringLiteral")) { - return new Identifier(loc, node.get("text").getAsString()); - } else { - return null; - } - } - - private Node convertEnumMember(JsonObject node, SourceLocation loc) throws ParseError { - Identifier name = convertNodeAsIdentifier(node.get("name").getAsJsonObject()); - if (name == null) - return null; - EnumMember member = new EnumMember(loc, name, convertChild(node, "initializer")); - attachSymbolInformation(member, node); - return member; - } - - private Node convertExportAssignment(JsonObject node, SourceLocation loc) throws ParseError { - if (hasChild(node, "isExportEquals") && node.get("isExportEquals").getAsBoolean()) - return new ExportWholeDeclaration(loc, convertChild(node, "expression")); - return new ExportDefaultDeclaration(loc, convertChild(node, "expression")); - } - - private Node convertExportDeclaration(JsonObject node, SourceLocation loc) throws ParseError { - Literal source = tryConvertChild(node, "moduleSpecifier", Literal.class); - if (hasChild(node, "exportClause")) { - return new ExportNamedDeclaration(loc, null, convertChildren(node.get("exportClause").getAsJsonObject(), "elements"), source); - } else { - return new ExportAllDeclaration(loc, source); - } - } - - private Node convertExportSpecifier(JsonObject node, SourceLocation loc) throws ParseError { - return new ExportSpecifier(loc, convertChild(node, hasChild(node, "propertyName") ? "propertyName" : "name"), convertChild(node, "name")); - } - - private Node convertExpressionStatement(JsonObject node, - SourceLocation loc) throws ParseError { - Expression expression = convertChild(node, "expression"); - return new ExpressionStatement(loc, expression); - } - - private Node convertExpressionWithTypeArguments(JsonObject node, SourceLocation loc) throws ParseError { - Expression expression = convertChild(node, "expression"); - List typeArguments = convertChildrenAsTypes(node, "typeArguments"); - if (typeArguments.isEmpty()) - return expression; - return new ExpressionWithTypeArguments(loc, expression, typeArguments); - } - - private Node convertExternalModuleReference(JsonObject node, SourceLocation loc) throws ParseError { - return new ExternalModuleReference(loc, convertChild(node, "expression")); - } - - private Node convertFalseKeyword(SourceLocation loc) { - return new Literal(loc, TokenType._false, false); - } - - private Node convertNumericLiteral(JsonObject node, SourceLocation loc) - throws NumberFormatException { - return new Literal(loc, TokenType.num, Double.valueOf(node.get("text").getAsString())); - } - - private Node convertForStatement(JsonObject node, SourceLocation loc) throws ParseError { - return new ForStatement(loc, convertChild(node, "initializer"), convertChild(node, "condition"), convertChild(node, "incrementor"), convertChild(node, "statement")); - } - - private Node convertForInStatement(JsonObject node, SourceLocation loc) throws ParseError { - Node initializer = convertChild(node, "initializer"); - if (initializer instanceof Expression) - initializer = convertLValue((Expression) initializer); - return new ForInStatement(loc, initializer, convertChild(node, "expression"), convertChild(node, "statement"), false); - } - - private Node convertForOfStatement(JsonObject node, SourceLocation loc) throws ParseError { - Node initializer = convertChild(node, "initializer"); - if (initializer instanceof Expression) - initializer = convertLValue((Expression) initializer); - return new ForOfStatement(loc, initializer, convertChild(node, "expression"), convertChild(node, "statement")); - } - - private Node convertFunctionDeclaration(JsonObject node, - SourceLocation loc) throws ParseError { - List params = convertParameters(node); - Identifier fnId = convertChild(node, "name", "Identifier"); - BlockStatement fnbody = convertChild(node, "body"); - boolean generator = hasChild(node, "asteriskToken"); - boolean async = hasModifier(node, "AsyncKeyword"); - boolean hasDeclareKeyword = hasModifier(node, "DeclareKeyword"); - List paramTypes = convertParameterTypes(node); - List typeParameters = convertChildrenNotNull(node, "typeParameters"); - ITypeExpression returnType = convertChildAsType(node, "type"); - ITypeExpression thisParam = convertThisParameterType(node); - FunctionDeclaration function = new FunctionDeclaration(loc, fnId, params, fnbody, generator, async, hasDeclareKeyword, - typeParameters, paramTypes, returnType, thisParam); - attachSymbolInformation(function, node); - return fixExports(loc, function); - } - - private Node convertFunctionExpression(JsonObject node, - SourceLocation loc) throws ParseError { - Identifier fnId = convertChild(node, "name", "Identifier"); - List params = convertParameters(node); - BlockStatement fnbody = convertChild(node, "body"); - boolean generator = hasChild(node, "asteriskToken"); - boolean async = hasModifier(node, "AsyncKeyword"); - List paramTypes = convertParameterTypes(node); - List paramDecorators = convertParameterDecorators(node); - ITypeExpression returnType = convertChildAsType(node, "type"); - ITypeExpression thisParam = convertThisParameterType(node); - return new FunctionExpression(loc, fnId, params, fnbody, generator, async, convertChildrenNotNull(node, "typeParameters"), - paramTypes, paramDecorators, returnType, thisParam); - } - - private Node convertFunctionType(JsonObject node, SourceLocation loc) throws ParseError { - return new FunctionTypeExpr(loc, convertImplicitFunction(node, loc), false); - } - - /** - * Gets the original text out of an Identifier's "escapedText" field. - */ - private String unescapeLeadingUnderscores(String text) { - // The TypeScript compiler inserts an additional underscore in front of - // identifiers that begin with two underscores. - if (text.startsWith("___")) { - return text.substring(1); - } else { - return text; - } - } - - /** Returns the contents of the given identifier as a string. */ - private String getIdentifierText(JsonObject identifierNode) { - if (identifierNode.has("text")) - return identifierNode.get("text").getAsString(); - else - return unescapeLeadingUnderscores(identifierNode.get("escapedText").getAsString()); - } - - private Identifier convertIdentifier(JsonObject node, SourceLocation loc) { - Identifier id = new Identifier(loc, getIdentifierText(node)); - attachSymbolInformation(id, node); - return id; - } - - private Node convertKeywordTypeExpr(JsonObject node, SourceLocation loc, String text) { - return new KeywordTypeExpr(loc, text); - } - - private Node convertUnionType(JsonObject node, SourceLocation loc) throws ParseError { - return new UnionTypeExpr(loc, convertChildrenAsTypes(node, "types")); - } - - private Node convertIfStatement(JsonObject node, SourceLocation loc) throws ParseError { - return new IfStatement(loc, convertChild(node, "expression"), convertChild(node, "thenStatement"), convertChild(node, "elseStatement")); - } - - private Node convertImportClause(JsonObject node, SourceLocation loc) throws ParseError { - return new ImportDefaultSpecifier(loc, convertChild(node, "name")); - } - - private Node convertImportDeclaration(JsonObject node, SourceLocation loc) throws ParseError { - Literal src = tryConvertChild(node, "moduleSpecifier", Literal.class); - List specifiers = new ArrayList<>(); - if (hasChild(node, "importClause")) { - JsonObject importClause = node.get("importClause").getAsJsonObject(); - if (hasChild(importClause, "name")) { - specifiers.add(convertChild(node, "importClause")); - } - if (hasChild(importClause, "namedBindings")) { - JsonObject namedBindings = importClause.get("namedBindings").getAsJsonObject(); - if (hasKind(namedBindings, "NamespaceImport")) { - specifiers.add(convertChild(importClause, "namedBindings")); - } else { - specifiers.addAll(convertChildren(namedBindings, "elements")); - } - } - } - return new ImportDeclaration(loc, specifiers, src); - } - - private Node convertImportEqualsDeclaration(JsonObject node, SourceLocation loc) throws ParseError { - return fixExports(loc, new ImportWholeDeclaration(loc, convertChild(node, "name"), convertChild(node, "moduleReference"))); - } - - private Node convertImportKeyword(SourceLocation loc) { - return new Identifier(loc, "import"); - } - - private Node convertImportSpecifier(JsonObject node, SourceLocation loc) throws ParseError { - boolean hasImported = hasChild(node, "propertyName"); - Identifier imported = convertChild(node, hasImported ? "propertyName" : "name"); - Identifier local = convertChild(node, "name"); - return new ImportSpecifier(loc, imported, local); - } - - private Node convertImportType(JsonObject node, SourceLocation loc) throws ParseError { - // This is a type such as `import("./foo").bar.Baz`. - // - // The TypeScript AST represents import types as the root of a qualified name, - // whereas we represent them as the leftmost qualifier. - // - // So in our AST, ImportTypeExpr just represents `import("./foo")`, and `.bar.Baz` - // is represented by nested MemberExpr nodes. - // - // Additionally, an import type can be prefixed by `typeof`, such as `typeof import("foo")`. - // We convert these to TypeofTypeExpr. - - // Get the source range of the `import(path)` part. - Position importStart = loc.getStart(); - Position importEnd = loc.getEnd(); - boolean isTypeof = false; - if (node.has("isTypeOf") && node.get("isTypeOf").getAsBoolean() == true) { - isTypeof = true; - Matcher m = TYPEOF_START.matcher(loc.getSource()); - if (m.find()) { - importStart = advance(importStart, m.group(0)); - } - } - // Find the ending parenthesis in `import(path)` by skipping whitespace after `path`. - ITypeExpression path = convertChild(node, "argument"); - String endSrc = loc.getSource().substring(path.getLoc().getEnd().getOffset() - loc.getStart().getOffset()); - Matcher m = WHITESPACE_END_PAREN.matcher(endSrc); - if (m.find()) { - importEnd = advance(path.getLoc().getEnd(), m.group(0)); - } - SourceLocation importLoc = getSourceRange(importStart, importEnd); - ImportTypeExpr imprt = new ImportTypeExpr(importLoc, path); - - ITypeExpression typeName = buildQualifiedTypeAccess(imprt, (JsonObject) node.get("qualifier")); - if (isTypeof) { - return new TypeofTypeExpr(loc, typeName); - } - - List typeArguments = convertChildrenAsTypes(node, "typeArguments"); - if (!typeArguments.isEmpty()) { - return new GenericTypeExpr(loc, typeName, typeArguments); - } - return (Node) typeName; - } - - /** - * Converts the given JSON to a qualified name with `root` as the base. - * - * For example, `a.b.c` is converted to the AST corresponding to `root.a.b.c`. - */ - private ITypeExpression buildQualifiedTypeAccess(ITypeExpression root, JsonObject node) throws ParseError { - if (node == null) { - return root; - } - String kind = getKind(node); - ITypeExpression base; - Expression name; - if (kind == null || kind.equals("Identifier")) { - base = root; - name = convertIdentifier(node, getSourceLocation(node)); - } else if (kind.equals("QualifiedName")) { - base = buildQualifiedTypeAccess(root, (JsonObject) node.get("left")); - name = convertChild(node, "right"); - } else { - throw new ParseError("Unsupported syntax in import type", getSourceLocation(node).getStart()); - } - MemberExpression member = new MemberExpression(getSourceLocation(node), (Expression) base, name, false, false, false); - attachSymbolInformation(member, node); - return member; - } - - private Node convertIndexSignature(JsonObject node, SourceLocation loc) throws ParseError { - FunctionExpression function = convertImplicitFunction(node, loc); - int flags = getMemberModifierKeywords(node) | DeclarationFlags.abstract_; - return new MethodDefinition(loc, flags, Kind.INDEX_SIGNATURE, null, function); - } - - private Node convertIndexedAccessType(JsonObject node, SourceLocation loc) throws ParseError { - return new IndexedAccessTypeExpr(loc, convertChildAsType(node, "objectType"), convertChildAsType(node, "indexType")); - } - - private Node convertInferType(JsonObject node, SourceLocation loc) throws ParseError { - return new InferTypeExpr(loc, convertChild(node, "typeParameter")); - } - - private Node convertInterfaceDeclaration(JsonObject node, SourceLocation loc) throws ParseError { - Identifier name = convertChild(node, "name"); - List typeParameters = convertChildrenNotNull(node, "typeParameters"); - List> members = convertChildren(node, "members"); - List superInterfaces = null; - for (JsonElement elt : getChildIterable(node, "heritageClauses")) { - JsonObject heritageClause = elt.getAsJsonObject(); - if (heritageClause.get("token").getAsInt() == syntaxKindExtends) { - superInterfaces = convertSuperInterfaceClause(heritageClause.get("types").getAsJsonArray()); - break; - } - } - if (superInterfaces == null) { - superInterfaces = new ArrayList<>(); - } - InterfaceDeclaration iface = new InterfaceDeclaration(loc, name, typeParameters, superInterfaces, members); - attachSymbolInformation(iface, node); - return fixExports(loc, iface); - } - - private Node convertIntersectionType(JsonObject node, SourceLocation loc) throws ParseError { - return new IntersectionTypeExpr(loc, convertChildrenAsTypes(node, "types")); - } - - private Node convertJsxAttribute(JsonObject node, SourceLocation loc) throws ParseError { - return new JSXAttribute(loc, convertJSXName(convertChild(node, "name")), convertChild(node, "initializer")); - } - - private Node convertJsxClosingElement(JsonObject node, SourceLocation loc) throws ParseError { - return new JSXClosingElement(loc, convertJSXName(convertChild(node, "tagName"))); - } - - private Node convertJsxElement(JsonObject node, SourceLocation loc) throws ParseError { - return new JSXElement(loc, convertChild(node, "openingElement"), convertChildren(node, "children"), convertChild(node, "closingElement")); - } - - private Node convertJsxExpression(JsonObject node, SourceLocation loc) throws ParseError { - if (hasChild(node, "expression")) - return new JSXExpressionContainer(loc, convertChild(node, "expression")); - return new JSXExpressionContainer(loc, new JSXEmptyExpression(loc)); - } - - private Node convertJsxFragment(JsonObject node, SourceLocation loc) throws ParseError { - return new JSXElement(loc, convertChild(node, "openingFragment"), convertChildren(node, "children"), - convertChild(node, "closingFragment")); - } - - private Node convertJsxOpeningFragment(JsonObject node, SourceLocation loc) { - return new JSXOpeningElement(loc, null, Collections.emptyList(), false); - } - - private Node convertJsxClosingFragment(JsonObject node, SourceLocation loc) { - return new JSXClosingElement(loc, null); - } - - private List convertJsxAttributes(JsonObject node) throws ParseError { - JsonElement attributes = node.get("attributes"); - List convertedAttributes; - if (attributes.isJsonArray()) { - convertedAttributes = convertNodes(attributes.getAsJsonArray()); - } else { - convertedAttributes = convertChildren(attributes.getAsJsonObject(), "properties"); - } - return convertedAttributes; - } - - private Node convertJsxOpeningElement(JsonObject node, SourceLocation loc) throws ParseError { - List convertedAttributes = convertJsxAttributes(node); - return new JSXOpeningElement(loc, convertJSXName(convertChild(node, "tagName")), convertedAttributes, hasChild(node, "selfClosing")); - } - - private Node convertJsxSelfClosingElement(JsonObject node, SourceLocation loc) throws ParseError { - List convertedAttributes = convertJsxAttributes(node); - JSXOpeningElement opening = new JSXOpeningElement(loc, convertJSXName(convertChild(node, "tagName")), convertedAttributes, true); - return new JSXElement(loc, opening, new ArrayList<>(), null); - } - - private Node convertJsxSpreadAttribute(JsonObject node, - SourceLocation loc) throws ParseError { - return new JSXSpreadAttribute(loc, convertChild(node, "expression")); - } - - private Node convertJsxText(JsonObject node, SourceLocation loc) { - String text; - if (hasChild(node, "text")) - text = node.get("text").getAsString(); - else - text = ""; - return new Literal(loc, TokenType.string, text); - } - - private Node convertLabeledStatement(JsonObject node, SourceLocation loc) throws ParseError { - return new LabeledStatement(loc, convertChild(node, "label"), convertChild(node, "statement")); - } - - private Node convertLiteralType(JsonObject node, SourceLocation loc) throws ParseError { - return convertChild(node, "literal"); - } - - private Node convertMappedType(JsonObject node, SourceLocation loc) throws ParseError { - return new MappedTypeExpr(loc, convertChild(node, "typeParameter"), convertChildAsType(node, "type")); - } - - private Node convertMetaProperty(JsonObject node, SourceLocation loc) throws ParseError { - Position metaStart = loc.getStart(); - Position metaEnd = new Position(metaStart.getLine(), metaStart.getColumn()+3, metaStart.getOffset()+3); - SourceLocation metaLoc = new SourceLocation("new", metaStart, metaEnd); - Identifier meta = new Identifier(metaLoc, "new"); - return new MetaProperty(loc, meta, convertChild(node, "name")); - } - - private Node convertMethodDeclaration(JsonObject node, String kind, - SourceLocation loc) throws ParseError { - int flags = getMemberModifierKeywords(node); - if (hasComputedName(node)) { - flags |= DeclarationFlags.computed; - } - if (kind.equals("MethodSignature")) { - flags |= DeclarationFlags.abstract_; - } - MethodDefinition.Kind methodKind; - if ("GetAccessor".equals(kind)) - methodKind = Kind.GET; - else if ("SetAccessor".equals(kind)) - methodKind = Kind.SET; - else - methodKind = Kind.METHOD; - FunctionExpression method = convertImplicitFunction(node, loc); - MethodDefinition methodDefinition = new MethodDefinition(loc, flags, methodKind, convertChild(node, "name"), method); - if (node.has("decorators")) { - methodDefinition.addDecorators(convertChildren(node, "decorators")); - advanceUntilAfter(loc, methodDefinition.getDecorators()); - } - return methodDefinition; - } - - private FunctionExpression convertImplicitFunction(JsonObject node, SourceLocation loc) throws ParseError { - ITypeExpression returnType = convertChildAsType(node, "type"); - List paramTypes = convertParameterTypes(node); - List paramDecorators = convertParameterDecorators(node); - List typeParameters = convertChildrenNotNull(node, "typeParameters"); - FunctionExpression method = new FunctionExpression(loc, null, convertParameters(node), convertChild(node, "body"), - hasChild(node, "asteriskToken"), hasModifier(node, "AsyncKeyword"), typeParameters, paramTypes, - paramDecorators, returnType, null); - attachSymbolInformation(method, node); - return method; - } - - private Node convertNamespaceDeclaration(JsonObject node, SourceLocation loc) throws ParseError { - Node nameNode = convertChild(node, "name"); - List body; - Statement b = convertChild(node, "body"); - if (b instanceof BlockStatement) { - body = ((BlockStatement) b).getBody(); - } else { - body = new ArrayList<>(); - body.add(b); - } - if (nameNode instanceof Literal) { - // Declaration of form: declare module "X" {...} - return new ExternalModuleDeclaration(loc, (Literal) nameNode, body); - } - if (hasFlag(node, "GlobalAugmentation")) { - // Declaration of form: declare global {...} - return new GlobalAugmentationDeclaration(loc, body); - } - Identifier name = (Identifier) nameNode; - boolean isInstantiated = false; - for (Statement stmt : body) { - isInstantiated = isInstantiated || isInstantiatingNamespaceMember(stmt); - } - boolean hasDeclareKeyword = hasModifier(node, "DeclareKeyword"); - NamespaceDeclaration decl = new NamespaceDeclaration(loc, name, body, isInstantiated, hasDeclareKeyword); - attachSymbolInformation(decl, node); - if (hasFlag(node, "NestedNamespace")) { - // In a nested namespace declaration `namespace A.B`, the nested namespace `B` - // is implicitly exported. - return new ExportNamedDeclaration(loc, decl, new ArrayList<>(), null); - } else { - return fixExports(loc, decl); - } - } - - private boolean isInstantiatingNamespaceMember(Statement node) { - if (node instanceof ExportNamedDeclaration) { - // Ignore 'export' modifiers. - return isInstantiatingNamespaceMember(((ExportNamedDeclaration) node).getDeclaration()); - } - if (node instanceof NamespaceDeclaration) { - return ((NamespaceDeclaration) node).isInstantiated(); - } - if (node instanceof InterfaceDeclaration) { - return false; - } - if (node instanceof TypeAliasDeclaration) { - return false; - } - return true; - } - - private Node convertModuleBlock(JsonObject node, SourceLocation loc) throws ParseError { - return convertBlock(node, loc); - } - - private Node convertNamespaceExportDeclaration(JsonObject node, SourceLocation loc) throws ParseError { - return new ExportAsNamespaceDeclaration(loc, convertChild(node, "name")); - } - - private Node convertNamespaceImport(JsonObject node, SourceLocation loc) throws ParseError { - return new ImportNamespaceSpecifier(loc, convertChild(node, "name")); - } - - private Node convertNewExpression(JsonObject node, SourceLocation loc) throws ParseError { - List arguments; - if (hasChild(node, "arguments")) - arguments = convertChildren(node, "arguments"); - else - arguments = new ArrayList<>(); - List typeArguments = convertChildrenAsTypes(node, "typeArguments"); - NewExpression result = new NewExpression(loc, convertChild(node, "expression"), typeArguments, arguments); - attachResolvedSignature(result, node); - return result; - } - - private Node convertNonNullExpression(JsonObject node, SourceLocation loc) throws ParseError { - return new NonNullAssertion(loc, convertChild(node, "expression")); - } - - private Node convertNoSubstitutionTemplateLiteral(JsonObject node, - SourceLocation loc) { - List quasis = new ArrayList<>(); - TemplateElement elm = new TemplateElement(loc, node.get("text").getAsString(), loc.getSource().substring(1, loc.getSource().length()-1), true); - quasis.add(elm); - attachStaticType(elm, node); - return new TemplateLiteral(loc, new ArrayList<>(), quasis); - } - - private Node convertNullKeyword(SourceLocation loc) { - return new Literal(loc, TokenType._null, null); - } - - private Node convertObjectBindingPattern(JsonObject node, - SourceLocation loc) throws ParseError { - List properties = new ArrayList<>(); - for (JsonElement elt : node.get("elements").getAsJsonArray()) { - JsonObject element = elt.getAsJsonObject(); - SourceLocation eltLoc = getSourceLocation(element); - Expression propKey = hasChild(element, "propertyName") ? convertChild(element, "propertyName") : convertChild(element, "name"); - Expression propVal; - if (hasChild(element, "dotDotDotToken")) { - propVal = new RestElement(eltLoc, propKey); - } else if (hasChild(element, "initializer")) { - propVal = new AssignmentPattern(eltLoc, "=", convertChild(element, "name"), convertChild(element, "initializer")); - } else { - propVal = convertChild(element, "name"); - } - properties.add(new Property(eltLoc, propKey, propVal, "init", hasComputedName(element, "propertyName"), false)); - } - return new ObjectPattern(loc, properties); - } - - private Node convertObjectLiteralExpression(JsonObject node, - SourceLocation loc) throws ParseError { - List properties; - properties = new ArrayList(); - for (INode e : convertChildren(node, "properties")) { - if (e instanceof SpreadElement) { - properties.add(new Property(e.getLoc(), null, (Expression)e, Property.Kind.INIT.name(), false, false)); - } else if (e instanceof MethodDefinition) { - MethodDefinition md = (MethodDefinition) e; - Property.Kind kind = Property.Kind.INIT; - if (md.getKind() == Kind.GET) { - kind = Property.Kind.GET; - } else if (md.getKind() == Kind.SET) { - kind = Property.Kind.SET; - } - properties.add(new Property(e.getLoc(), md.getKey(), md.getValue(), kind.name(), md.isComputed(), true)); - } else { - properties.add((Property)e); - } - } - return new ObjectExpression(loc, properties); - } - - private Node convertOmittedExpression() { - return null; - } - - private Node convertOptionalType(JsonObject node, SourceLocation loc) throws ParseError { - return new OptionalTypeExpr(loc, convertChild(node, "type")); - } - - private ITypeExpression asType(Node node) { - return node instanceof ITypeExpression ? (ITypeExpression) node : null; - } - - private List convertChildrenAsTypes(JsonObject node, String child) throws ParseError { - List result = new ArrayList<>(); - JsonElement children = node.get(child); - if (!(children instanceof JsonArray)) - return result; - for (JsonElement childNode : children.getAsJsonArray()) { - ITypeExpression type = asType(convertNode(childNode.getAsJsonObject())); - if (type != null) - result.add(type); - } - return result; - } - - private ITypeExpression convertChildAsType(JsonObject node, String child) throws ParseError { - return asType(convertChild(node, child)); - } - - /** - * True if the given node is an Identifier node. - */ - private boolean isIdentifier(JsonElement node) { - if (node == null) - return false; - JsonObject object = node.getAsJsonObject(); - if (object == null) - return false; - String kind = getKind(object); - return kind == null || kind.equals("Identifier"); - } - - /** - * Returns true if this is the JSON object for the special "this" parameter. - *

      - * It should be given the JSON object of kind "Parameter". - */ - private boolean isThisParameter(JsonElement parameter) { - JsonObject name = parameter.getAsJsonObject().get("name").getAsJsonObject(); - return isIdentifier(name) && getIdentifierText(name).equals("this"); - } - - /** - * Returns the parameters of the given function, omitting the special "this" - * parameter, which we do not consider to be a proper parameter. - */ - private Iterable getProperParameters(JsonObject function) { - if (!function.has("parameters")) - return Collections.emptyList(); - JsonArray parameters = function.get("parameters").getAsJsonArray(); - if (parameters.size() > 0 && isThisParameter(parameters.get(0))) { - return CollectionUtil.skipIterable(parameters, 1); - } else { - return parameters; - } - } - - /** - * Returns the special "this" parameter of the given function, or {@code null} - * if the function does not declare a "this" parameter. - */ - private ITypeExpression convertThisParameterType(JsonObject function) throws ParseError { - if (!function.has("parameters")) - return null; - JsonArray parameters = function.get("parameters").getAsJsonArray(); - if (parameters.size() > 0 && isThisParameter(parameters.get(0))) { - return convertChildAsType(parameters.get(0).getAsJsonObject(), "type"); - } else { - return null; - } - } - - private List convertParameters(JsonObject function) throws ParseError { - return convertNodes(getProperParameters(function), true); - } - - private List convertParameterTypes(JsonObject function) throws ParseError { - List result = new ArrayList<>(); - for (JsonElement param : getProperParameters(function)) { - result.add(convertChildAsType(param.getAsJsonObject(), "type")); - } - return result; - } - - private List convertParameterFields(JsonObject function) throws ParseError { - List result = new ArrayList<>(); - int index = -1; - for (JsonElement paramElm : getProperParameters(function)) { - ++index; - JsonObject param = paramElm.getAsJsonObject(); - int flags = getMemberModifierKeywords(param); - if (flags == DeclarationFlags.none) { - // If there are no flags, this is not a field parameter. - continue; - } - // We generate a synthetic field node, but do not copy any of the AST nodes from - // the parameter. The QL library overrides accessors to the name and type - // annotation to return those from the corresponding parameter. - SourceLocation loc = getSourceLocation(param); - if (param.has("initializer")) { - // Do not include the default parameter value in the source range for the field. - SourceLocation endLoc; - if (param.has("type")) { - endLoc = getSourceLocation(param.get("type").getAsJsonObject()); - } else { - endLoc = getSourceLocation(param.get("name").getAsJsonObject()); - } - loc.setEnd(endLoc.getEnd()); - loc.setSource(source.substring(loc.getStart().getOffset(), loc.getEnd().getOffset())); - } - FieldDefinition field = new FieldDefinition(loc, flags, null, null, null, index); - result.add(field); - } - return result; - } - - private Node convertParameter(JsonObject node, SourceLocation loc) throws ParseError { - // Note that type annotations are not extracted in this function, but in a - // separate pass in convertParameterTypes above. - Expression name = convertChild(node, "name", "Identifier"); - if (hasChild(node, "dotDotDotToken")) - return new RestElement(loc, name); - if (hasChild(node, "initializer")) - return new AssignmentPattern(loc, "=", name, convertChild(node, "initializer")); - return name; - } - - private Node convertParenthesizedExpression(JsonObject node, - SourceLocation loc) throws ParseError { - return new ParenthesizedExpression(loc, convertChild(node, "expression")); - } - - private Node convertParenthesizedType(JsonObject node, SourceLocation loc) throws ParseError { - return new ParenthesizedTypeExpr(loc, convertChildAsType(node, "type")); - } - - private Node convertPostfixUnaryExpression(JsonObject node, - SourceLocation loc) throws ParseError { - String operator = getOperator(node); - return new UpdateExpression(loc, operator, convertChild(node, "operand"), false); - } - - private Node convertPrefixUnaryExpression(JsonObject node, - SourceLocation loc) throws ParseError { - String operator = getOperator(node); - if ("++".equals(operator) || "--".equals(operator)) - return new UpdateExpression(loc, operator, convertChild(node, "operand"), true); - else - return new UnaryExpression(loc, operator, convertChild(node, "operand"), true); - } - - private String getOperator(JsonObject node) throws ParseError { - int operatorId = node.get("operator").getAsInt(); - switch (syntaxKindMap.get(operatorId)) { - case "PlusPlusToken": - return "++"; - case "MinusMinusToken": - return "--"; - case "PlusToken": - return "+"; - case "MinusToken": - return "-"; - case "TildeToken": - return "~"; - case "ExclamationToken": - return "!"; - default: - throw new ParseError("Unsupported TypeScript operator " + operatorId, getSourceLocation(node).getStart()); - } - } - - private Node convertPropertyAccessExpression(JsonObject node, - SourceLocation loc) throws ParseError { - return new MemberExpression(loc, convertChild(node, "expression"), convertChild(node, "name"), false, false, false); - } - - private Node convertPropertyAssignment(JsonObject node, - SourceLocation loc) throws ParseError { - return new Property(loc, convertChild(node, "name"), convertChild(node, "initializer"), "init", hasComputedName(node), false); - } - - private Node convertPropertyDeclaration(JsonObject node, String kind, - SourceLocation loc) throws ParseError { - int flags = getMemberModifierKeywords(node); - if (hasComputedName(node)) { - flags |= DeclarationFlags.computed; - } - if (kind.equals("PropertySignature")) { - flags |= DeclarationFlags.abstract_; - } - if (node.get("questionToken") != null) { - flags |= DeclarationFlags.optional; - } - if (node.get("exclamationToken") != null) { - flags |= DeclarationFlags.definiteAssignmentAssertion; - } - FieldDefinition fieldDefinition = new FieldDefinition(loc, flags, convertChild(node, "name"), - convertChild(node, "initializer"), convertChildAsType(node, "type")); - if (node.has("decorators")) { - fieldDefinition.addDecorators(convertChildren(node, "decorators")); - advanceUntilAfter(loc, fieldDefinition.getDecorators()); - } - return fieldDefinition; - } - - private Node convertRegularExpressionLiteral(SourceLocation loc) { - return new Literal(loc, TokenType.regexp, null); - } - - private Node convertRestType(JsonObject node, SourceLocation loc) throws ParseError { - return new RestTypeExpr(loc, convertChild(node, "type")); - } - - private Node convertQualifiedName(JsonObject node, SourceLocation loc) throws ParseError { - MemberExpression expr = new MemberExpression(loc, convertChild(node, "left"), convertChild(node, "right"), false, false, false); - attachSymbolInformation(expr, node); - return expr; - } - - private Node convertReturnStatement(JsonObject node, SourceLocation loc) throws ParseError { - return new ReturnStatement(loc, convertChild(node, "expression")); - } - - private Node convertSemicolonClassElement() { - return null; - } - - private Node convertSourceFile(JsonObject node, SourceLocation loc) throws ParseError { - List statements = convertNodes(node.get("statements").getAsJsonArray()); - Program program = new Program(loc, statements, "module"); - attachSymbolInformation(program, node); - return program; - } - - private Node convertShorthandPropertyAssignment(JsonObject node, - SourceLocation loc) throws ParseError { - return new Property(loc, convertChild(node, "name"), convertChild(node, "name"), "init", false, false); - } - - private Node convertSpreadElement(JsonObject node, SourceLocation loc) throws ParseError { - return new SpreadElement(loc, convertChild(node, "expression")); - } - - private Node convertStringLiteral(JsonObject node, SourceLocation loc) { - return new Literal(loc, TokenType.string, node.get("text").getAsString()); - } - - private Node convertSuperKeyword(SourceLocation loc) { - return new Super(loc); - } - - private Node convertSwitchStatement(JsonObject node, SourceLocation loc) throws ParseError { - JsonObject caseBlock = node.get("caseBlock").getAsJsonObject(); - return new SwitchStatement(loc, convertChild(node, "expression"), convertChildren(caseBlock, "clauses")); - } - - private Node convertTaggedTemplateExpression(JsonObject node, - SourceLocation loc) throws ParseError { - return new TaggedTemplateExpression(loc, convertChild(node, "tag"), convertChild(node, "template")); - } - - private Node convertTemplateExpression(JsonObject node, - SourceLocation loc) throws ParseError { - List quasis; - List expressions = new ArrayList<>(); - quasis = new ArrayList<>(); - quasis.add(convertChild(node, "head")); - for (JsonElement elt : node.get("templateSpans").getAsJsonArray()) { - JsonObject templateSpan = (JsonObject) elt; - expressions.add(convertChild(templateSpan, "expression")); - quasis.add(convertChild(templateSpan, "literal")); - } - return new TemplateLiteral(loc, expressions, quasis); - } - - private Node convertTemplateElement(JsonObject node, String kind, - SourceLocation loc) { - boolean tail = "TemplateTail".equals(kind); - if (loc.getSource().startsWith("`") || loc.getSource().startsWith("}")) { - loc.setSource(loc.getSource().substring(1)); - Position start = loc.getStart(); - loc.setStart(new Position(start.getLine(), start.getColumn()+1, start.getColumn()+1)); - } - if (loc.getSource().endsWith("${")) { - loc.setSource(loc.getSource().substring(0, loc.getSource().length()-2)); - Position end = loc.getEnd(); - loc.setEnd(new Position(end.getLine(), end.getColumn()-2, end.getColumn()-2)); - } - if (loc.getSource().endsWith("`")) { - loc.setSource(loc.getSource().substring(0, loc.getSource().length()-1)); - Position end = loc.getEnd(); - loc.setEnd(new Position(end.getLine(), end.getColumn()-1, end.getColumn()-1)); - } - return new TemplateElement(loc, node.get("text").getAsString(), loc.getSource(), tail); - } - - private Node convertThisKeyword(SourceLocation loc) { - return new ThisExpression(loc); - } - - private Node convertThrowStatement(JsonObject node, SourceLocation loc) throws ParseError { - Expression expr = convertChild(node, "expression"); - if (expr == null) - return convertEmptyStatement(loc); - return new ThrowStatement(loc, expr); - } - - private Node convertTrueKeyword(SourceLocation loc) { - return new Literal(loc, TokenType._true, true); - } - - private Node convertTryStatement(JsonObject node, SourceLocation loc) throws ParseError { - return new TryStatement(loc, convertChild(node, "tryBlock"), convertChild(node, "catchClause"), null, - convertChild(node, "finallyBlock")); - } - - private Node convertTupleType(JsonObject node, SourceLocation loc) throws ParseError { - return new TupleTypeExpr(loc, convertChildrenAsTypes(node, "elementTypes")); - } - - private Node convertTypeAliasDeclaration(JsonObject node, SourceLocation loc) throws ParseError { - TypeAliasDeclaration typeAlias = new TypeAliasDeclaration(loc, convertChild(node, "name"), - convertChildrenNotNull(node, "typeParameters"), convertChildAsType(node, "type")); - attachSymbolInformation(typeAlias, node); - return fixExports(loc, typeAlias); - } - - private Node convertTypeAssertionExpression(JsonObject node, SourceLocation loc) throws ParseError { - return new TypeAssertion(loc, convertChild(node, "expression"), convertChildAsType(node, "type"), false); - } - - private Node convertTypeLiteral(JsonObject obj, SourceLocation loc) throws ParseError { - return new InterfaceTypeExpr(loc, convertChildren(obj, "members")); - } - - private Node convertTypeOfExpression(JsonObject node, SourceLocation loc) throws ParseError { - return new UnaryExpression(loc, "typeof", convertChild(node, "expression"), true); - } - - private Node convertTypeOperator(JsonObject node, SourceLocation loc) throws ParseError { - String operator = syntaxKinds.get("" + node.get("operator").getAsInt()).getAsString(); - if (operator.equals("KeyOfKeyword")) { - return new KeyofTypeExpr(loc, convertChildAsType(node, "type")); - } - if (operator.equals("UniqueKeyword")) { - return new KeywordTypeExpr(loc, "unique symbol"); - } - throw new ParseError("Unsupported TypeScript syntax", loc.getStart()); - } - - private Node convertTypeParameter(JsonObject node, SourceLocation loc) throws ParseError { - return new TypeParameter(loc, convertChild(node, "name"), convertChildAsType(node, "constraint"), - convertChildAsType(node, "default")); - } - - private Node convertTypePredicate(JsonObject node, SourceLocation loc) throws ParseError { - return new IsTypeExpr(loc, convertChildAsType(node, "parameterName"), convertChildAsType(node, "type")); - } - - private Node convertTypeReference(JsonObject node, SourceLocation loc) throws ParseError { - ITypeExpression typeName = convertChild(node, "typeName"); - List typeArguments = convertChildrenAsTypes(node, "typeArguments"); - if (typeArguments.isEmpty()) - return (Node) typeName; - return new GenericTypeExpr(loc, typeName, typeArguments); - } - - private Node convertTypeQuery(JsonObject node, SourceLocation loc) throws ParseError { - return new TypeofTypeExpr(loc, convertChildAsType(node, "exprName")); - } - - private Node convertVariableDeclaration(JsonObject node, - SourceLocation loc) throws ParseError { - return new VariableDeclarator(loc, convertChild(node, "name"), convertChild(node, "initializer"), - convertChildAsType(node, "type"), DeclarationFlags.getDefiniteAssignmentAssertion(node.get("exclamationToken") != null)); - } - - private Node convertVariableDeclarationList(JsonObject node, - SourceLocation loc) throws ParseError { - return new VariableDeclaration(loc, getDeclarationKind(node), convertVariableDeclarations(node), false); - } - - private List convertVariableDeclarations(JsonObject node) throws ParseError { - if (node.get("declarations").getAsJsonArray().size() == 0) - throw new ParseError("Unexpected token", getSourceLocation(node).getEnd()); - return convertChildren(node, "declarations"); - } - - private Node convertVariableStatement(JsonObject node, SourceLocation loc) throws ParseError { - JsonObject declarationList = node.get("declarationList").getAsJsonObject(); - String declarationKind = getDeclarationKind(declarationList); - List declarations = convertVariableDeclarations(declarationList); - boolean hasDeclareKeyword = hasModifier(node, "DeclareKeyword"); - VariableDeclaration vd = new VariableDeclaration(loc, declarationKind, declarations, hasDeclareKeyword); - return fixExports(loc, vd); - } - - private Node convertVoidExpression(JsonObject node, SourceLocation loc) throws ParseError { - return new UnaryExpression(loc, "void", convertChild(node, "expression"), true); - } - - private Node convertWhileStatement(JsonObject node, SourceLocation loc) throws ParseError { - return new WhileStatement(loc, convertChild(node, "expression"), convertChild(node, "statement")); - } - - private Node convertWithStatement(JsonObject node, SourceLocation loc) throws ParseError { - return new WithStatement(loc, convertChild(node, "expression"), convertChild(node, "statement")); - } - - private Node convertYieldExpression(JsonObject node, SourceLocation loc) throws ParseError { - return new YieldExpression(loc, convertChild(node, "expression"), hasChild(node, "asteriskToken")); - } - - /** - * Convert {@code e} to an lvalue expression, replacing {@link ArrayExpression} with - * {@link ArrayPattern}, {@link AssignmentExpression} with {@link AssignmentPattern}, - * {@link ObjectExpression} with {@link ObjectPattern} and {@link SpreadElement} with - * {@link RestElement}. - */ - private Expression convertLValue(Expression e) { - if (e == null) - return null; - - SourceLocation loc = e.getLoc(); - if (e instanceof ArrayExpression) { - List elts = new ArrayList(); - for (Expression elt : ((ArrayExpression) e).getElements()) - elts.add(convertLValue(elt)); - return new ArrayPattern(loc, elts); - } - if (e instanceof AssignmentExpression) { - AssignmentExpression a = (AssignmentExpression)e; - return new AssignmentPattern(loc, a.getOperator(), convertLValue(a.getLeft()), a.getRight()); - } - if (e instanceof ObjectExpression) { - List props = new ArrayList(); - for (Property prop : ((ObjectExpression) e).getProperties()) { - Expression key = prop.getKey(); - Expression rawValue = prop.getRawValue(); - String kind = prop.getKind().name(); - boolean isComputed = prop.isComputed(); - boolean isMethod = prop.isMethod(); - props.add(new Property(prop.getLoc(), key, convertLValue(rawValue), kind, isComputed, isMethod)); - } - return new ObjectPattern(loc, props); - } - if (e instanceof ParenthesizedExpression) - return new ParenthesizedExpression(loc, convertLValue(((ParenthesizedExpression) e).getExpression())); - if (e instanceof SpreadElement) - return new RestElement(e.getLoc(), convertLValue(((SpreadElement) e).getArgument())); - return e; - } - - /** - * Convert {@code e} to an {@link IJSXName}. - */ - private IJSXName convertJSXName(Expression e) { - if (e instanceof Identifier) - return new JSXIdentifier(e.getLoc(), ((Identifier) e).getName()); - if (e instanceof MemberExpression) { - MemberExpression me = (MemberExpression) e; - return new JSXMemberExpression(e.getLoc(), convertJSXName(me.getObject()), (JSXIdentifier) convertJSXName(me.getProperty())); - } - if (e instanceof ThisExpression) - return new JSXIdentifier(e.getLoc(), "this"); - return (IJSXName) e; - } - - /** - * Check whether {@code decl} has an {@code export} annotation, and if so wrap - * it inside an {@link ExportDeclaration}. - *

      - * If the declared statement has decorators, the {@code loc} should first be - * advanced past these using {@link #advanceUntilAfter}. - */ - private Node fixExports(SourceLocation loc, Statement decl) { - Matcher m = EXPORT_DECL_START.matcher(loc.getSource()); - if (m.find()) { - String skipped = m.group(0); - SourceLocation outerLoc = new SourceLocation(loc.getSource(), loc.getStart(), loc.getEnd()); - advance(loc, skipped); - // capture group 1 is `default`, if present - if (m.group(1) == null) - return new ExportNamedDeclaration(outerLoc, decl, new ArrayList<>(), null); - return new ExportDefaultDeclaration(outerLoc, decl); - } - return decl; - } - - /** - * Holds if the {@code name} property of the given AST node is a computed - * property name. - */ - private boolean hasComputedName(JsonObject node) { - return hasComputedName(node, "name"); - } - - /** - * Holds if the given property of the given AST node is a computed property name. - */ - private boolean hasComputedName(JsonObject node, String propName) { - return hasKind(node.get(propName), "ComputedPropertyName"); - } - - /** - * Update the start position and source text of {@code loc} by skipping over the string - * {@code skipped}. - */ - private void advance(SourceLocation loc, String skipped) { - loc.setStart(advance(loc.getStart(), skipped)); - loc.setSource(loc.getSource().substring(skipped.length())); - } - - /** - * Update the start position of @{code loc} by skipping over the given children - * and any following whitespace and comments, provided they are contained in the - * source location. - */ - private void advanceUntilAfter(SourceLocation loc, List nodes) { - if (nodes.isEmpty()) - return; - INode last = nodes.get(nodes.size() - 1); - int offset = last.getLoc().getEnd().getOffset() - loc.getStart().getOffset(); - if (offset <= 0) - return; - offset += matchWhitespace(last.getLoc().getEnd().getOffset()).length(); - if (offset >= loc.getSource().length()) - return; - loc.setStart(advance(loc.getStart(), loc.getSource().substring(0, offset))); - loc.setSource(loc.getSource().substring(offset)); - } - - /** - * Get the longest sequence of whitespace or comment characters starting at the - * given offset. - */ - private String matchWhitespace(int offset) { - Matcher m = WHITESPACE.matcher(source.substring(offset)); - m.find(); - return m.group(0); - } - - /** - * Create a position corresponding to {@code pos}, but updated by skipping over the - * string {@code skipped}. - */ - private Position advance(Position pos, String skipped) { - int innerStartOffset = pos.getOffset() + skipped.length(); - int innerStartLine = pos.getLine(), innerStartColumn = pos.getColumn(); - Matcher m = LINE_TERMINATOR.matcher(skipped); - int lastEnd = 0; - while (m.find()) { - ++innerStartLine; - innerStartColumn = 1; - lastEnd = m.end(); - } - innerStartColumn += skipped.length() - lastEnd; - if (lastEnd > 0) - --innerStartColumn; - Position innerStart = new Position(innerStartLine, innerStartColumn, innerStartOffset); - return innerStart; - } - - /** - * Get the source location of the given AST node. - */ - private SourceLocation getSourceLocation(JsonObject node) { - Position start = getPosition(node.get("$pos")); - Position end = getPosition(node.get("$end")); - int startOffset = start.getOffset(); - int endOffset = end.getOffset(); - if (startOffset > endOffset) - startOffset = endOffset; - if (endOffset > source.length()) - endOffset = source.length(); - return new SourceLocation(source.substring(startOffset, endOffset), start, end); - } - - /** - * Convert the given position object into a {@link Position}. - * For start positions, we need to skip over whitespace, which is included in - * the positions reported by the TypeScript compiler. - */ - private Position getPosition(JsonElement elm) { - int offset = elm.getAsInt(); - int line = getLineFromPos(offset); - int column = getColumnFromLinePos(line, offset); - return new Position(line + 1, column, offset); - } - - private Iterable getModifiers(JsonObject node) { - JsonElement mods = node.get("modifiers"); - if (!(mods instanceof JsonArray)) - return Collections.emptyList(); - return (JsonArray) mods; - } - - /** - * Returns a specific modifier from the given node (or null if absent), - * as defined by its modifiers property and the kind property - * of the modifier AST node. - */ - private JsonObject getModifier(JsonObject node, String modKind) { - for (JsonElement mod : getModifiers(node)) - if (mod instanceof JsonObject) - if (hasKind((JsonObject) mod, modKind)) - return (JsonObject) mod; - return null; - } - - /** - * Check whether a node has a particular modifier, as defined by its - * modifiers property and the kind property of the modifier - * AST node. - */ - private boolean hasModifier(JsonObject node, String modKind) { - return getModifier(node, modKind) != null; - } - - private int getDeclarationModifierFromKeyword(String kind) { - switch (kind) { - case "AbstractKeyword": - return DeclarationFlags.abstract_; - case "StaticKeyword": - return DeclarationFlags.static_; - case "ReadonlyKeyword": - return DeclarationFlags.readonly; - case "PublicKeyword": - return DeclarationFlags.public_; - case "PrivateKeyword": - return DeclarationFlags.private_; - case "ProtectedKeyword": - return DeclarationFlags.protected_; - default: - return DeclarationFlags.none; - } - } - - /** - * Returns the set of member flags corresponding to the modifier keywords - * present on the given node. - */ - private int getMemberModifierKeywords(JsonObject node) { - int flags = DeclarationFlags.none; - for (JsonElement mod : getModifiers(node)) { - if (mod instanceof JsonObject) { - JsonObject modObject = (JsonObject) mod; - flags |= getDeclarationModifierFromKeyword(getKind(modObject)); - } - } - return flags; - } - - /** - * Check whether a node has a particular flag, as defined by its flags - * property and the ts.NodeFlags in enum. - */ - private boolean hasFlag(JsonObject node, String flagName) { - JsonElement flagDescriptor = this.nodeFlags.get(flagName); - if (flagDescriptor == null) { - throw new RuntimeException("Incompatible version of TypeScript installed. Missing node flag " + flagName); - } - int flagId = flagDescriptor.getAsInt(); - JsonElement flags = node.get("flags"); - if (flags instanceof JsonPrimitive) { - return (flags.getAsInt() & flagId) != 0; - } - return false; - } - - /** - * Gets the numeric value of the syntax kind enum with the given name. - */ - private int getSyntaxKind(String syntaxKind) { - JsonElement descriptor = this.syntaxKinds.get(syntaxKind); - if (descriptor == null) { - throw new RuntimeException("Incompatible version of TypeScript installed. Missing syntax kind " + syntaxKind); - } - return descriptor.getAsInt(); - } - - /** - * Check whether a node has a child with a given name. - */ - private boolean hasChild(JsonObject node, String prop) { - if (!node.has(prop)) - return false; - return !(node.get(prop) instanceof JsonNull); - } - - /** - * Returns an iterator over the elements of the given child array, or an empty - * iterator if the given child is not an array. - */ - private Iterable getChildIterable(JsonObject node, String child) { - JsonElement elt = node.get(child); - if (!(elt instanceof JsonArray)) - return Collections.emptyList(); - return (JsonArray) elt; - } - - /** - * Gets the kind of the given node. - */ - private String getKind(JsonElement node) { - if (node instanceof JsonObject) { - JsonElement kind = ((JsonObject) node).get("kind"); - if (kind instanceof JsonPrimitive && ((JsonPrimitive) kind).isNumber()) - return syntaxKindMap.get(kind.getAsInt()); - } - return null; - } - - /** - * Holds if the given node has the given kind. - */ - private boolean hasKind(JsonElement node, String kind) { - return kind.equals(getKind(node)); - } - - /** - * Gets the declaration kind of the given node, which is one of {@code "var"}, - * {@code "let"} or {@code "const"}. - */ - private String getDeclarationKind(JsonObject declarationList) { - return declarationList.get("$declarationKind").getAsString(); - } + private String source; + private final JsonObject nodeFlags; + private final JsonObject syntaxKinds; + private final Map nodeFlagMap = new LinkedHashMap<>(); + private final Map syntaxKindMap = new LinkedHashMap<>(); + private int[] lineStarts; + + private int syntaxKindExtends; + + private static final Pattern LINE_TERMINATOR = Pattern.compile("\\n|\\r\\n|\\r|\\u2028|\\u2029"); + private static final String WHITESPACE_CHAR = "(?:\\s|//.*|/\\*(?:[^*]|\\*(?!/))*\\*/)"; + private static final Pattern WHITESPACE = Pattern.compile("^" + WHITESPACE_CHAR + "*"); + private static final Pattern EXPORT_DECL_START = + Pattern.compile("^export" + "(" + WHITESPACE_CHAR + "+default)?" + WHITESPACE_CHAR + "+"); + private static final Pattern TYPEOF_START = Pattern.compile("^typeof" + WHITESPACE_CHAR + "+"); + private static final Pattern WHITESPACE_END_PAREN = + Pattern.compile("^" + WHITESPACE_CHAR + "*\\)"); + + TypeScriptASTConverter(JsonObject nodeFlags, JsonObject syntaxKinds) { + this.nodeFlags = nodeFlags; + this.syntaxKinds = syntaxKinds; + makeEnumIdMap(nodeFlags, nodeFlagMap); + makeEnumIdMap(syntaxKinds, syntaxKindMap); + this.syntaxKindExtends = getSyntaxKind("ExtendsKeyword"); + } + + /** Builds a mapping from ID to name given a TypeScript enum object. */ + private void makeEnumIdMap(JsonObject enumObject, Map idToName) { + for (Map.Entry entry : enumObject.entrySet()) { + JsonPrimitive prim = entry.getValue().getAsJsonPrimitive(); + if (prim.isNumber() && !idToName.containsKey(prim.getAsInt())) { + idToName.put(prim.getAsInt(), entry.getKey()); + } + } + } + + /** + * Convert the given TypeScript AST (which was parsed from {@code source}) into a parser {@link + * Result}. + */ + public Result convertAST(JsonObject ast, String source) { + this.lineStarts = toIntArray(ast.getAsJsonArray("$lineStarts")); + + List errors = new ArrayList(); + + // process parse diagnostics (i.e., syntax errors) reported by the TypeScript compiler + JsonArray parseDiagnostics = ast.get("parseDiagnostics").getAsJsonArray(); + if (parseDiagnostics.size() > 0) { + for (JsonElement elt : parseDiagnostics) { + JsonObject parseDiagnostic = elt.getAsJsonObject(); + String message = parseDiagnostic.get("messageText").getAsString(); + Position pos = getPosition(parseDiagnostic.get("$pos")); + errors.add(new ParseError(message, pos.getLine(), pos.getColumn(), pos.getOffset())); + } + return new Result(source, null, new ArrayList<>(), new ArrayList<>(), errors); + } + + this.source = source; + + List tokens = new ArrayList<>(); + List comments = new ArrayList<>(); + extractTokensAndComments(ast, tokens, comments); + Node converted; + try { + converted = convertNode(ast); + } catch (ParseError e) { + converted = null; + errors.add(e); + } + return new Result(source, converted, tokens, comments, errors); + } + + /** Converts a JSON array to an int array. The array is assumed to only contain integers. */ + private static int[] toIntArray(JsonArray array) { + int[] result = new int[array.size()]; + for (int i = 0; i < array.size(); ++i) { + result[i] = array.get(i).getAsInt(); + } + return result; + } + + private int getLineFromPos(int pos) { + int low = 0, high = this.lineStarts.length - 1; + while (low < high) { + int mid = high - ((high - low) >> 1); // Get middle, rounding up. + int startOfLine = lineStarts[mid]; + if (startOfLine <= pos) { + low = mid; + } else { + high = mid - 1; + } + } + return low; + } + + private int getColumnFromLinePos(int line, int pos) { + return pos - lineStarts[line]; + } + + /** Extract tokens and comments from the given TypeScript AST. */ + private void extractTokensAndComments( + JsonObject ast, List tokens, List comments) { + for (JsonElement elt : ast.get("$tokens").getAsJsonArray()) { + JsonObject token = elt.getAsJsonObject(); + String text = token.get("text").getAsString(); + Position start = getPosition(token.get("tokenPos")); + Position end = advance(start, text); + SourceLocation loc = new SourceLocation(text, start, end); + String kind = getKind(token); + switch (kind) { + case "EndOfFileToken": + tokens.add(new Token(loc, Token.Type.EOF)); + break; + case "SingleLineCommentTrivia": + case "MultiLineCommentTrivia": + String cookedText; + if (text.startsWith("//")) cookedText = text.substring(2); + else cookedText = text.substring(2, text.length() - 2); + comments.add(new Comment(loc, cookedText)); + break; + case "TemplateHead": + case "TemplateMiddle": + case "TemplateTail": + case "NoSubstitutionTemplateLiteral": + tokens.add(new Token(loc, Token.Type.STRING)); + break; + case "Identifier": + tokens.add(new Token(loc, Token.Type.NAME)); + break; + case "NumericLiteral": + tokens.add(new Token(loc, Token.Type.NUM)); + break; + case "StringLiteral": + tokens.add(new Token(loc, Token.Type.STRING)); + break; + case "RegularExpressionLiteral": + tokens.add(new Token(loc, Token.Type.REGEXP)); + break; + default: + Token.Type tp; + if (kind.endsWith("Token")) { + tp = Token.Type.PUNCTUATOR; + } else if (kind.endsWith("Keyword")) { + if (text.equals("null")) tp = Token.Type.NULL; + else if (text.equals("true")) tp = Token.Type.TRUE; + else if (text.equals("false")) tp = Token.Type.FALSE; + else tp = Token.Type.KEYWORD; + } else { + continue; + } + tokens.add(new Token(loc, tp)); + } + } + } + + /** Convert the given TypeScript node and its children into a JavaScript {@link Node}. */ + private Node convertNode(JsonObject node) throws ParseError { + return convertNode(node, null); + } + + /** + * Convert the given TypeScript node and its children into a JavaScript {@link Node}. If the + * TypesScript node has no explicit {@code kind}, it is assumed to be {@code defaultKind}. + */ + private Node convertNode(JsonObject node, String defaultKind) throws ParseError { + Node astNode = convertNodeUntyped(node, defaultKind); + attachStaticType(astNode, node); + return astNode; + } + + /** Helper method for `convertNode` that does everything except attaching type information. */ + private Node convertNodeUntyped(JsonObject node, String defaultKind) throws ParseError { + String kind = getKind(node); + if (kind == null) kind = defaultKind; + if (kind == null) kind = "Identifier"; + SourceLocation loc = getSourceLocation(node); + switch (kind) { + case "AnyKeyword": + return convertKeywordTypeExpr(node, loc, "any"); + case "ArrayBindingPattern": + return convertArrayBindingPattern(node, loc); + case "ArrayLiteralExpression": + return convertArrayLiteralExpression(node, loc); + case "ArrayType": + return convertArrayType(node, loc); + case "ArrowFunction": + return convertArrowFunction(node, loc); + case "AsExpression": + return convertAsExpression(node, loc); + case "AwaitExpression": + return convertAwaitExpression(node, loc); + case "BigIntKeyword": + return convertKeywordTypeExpr(node, loc, "bigint"); + case "BigIntLiteral": + return convertBigIntLiteral(node, loc); + case "BinaryExpression": + return convertBinaryExpression(node, loc); + case "Block": + return convertBlock(node, loc); + case "BooleanKeyword": + return convertKeywordTypeExpr(node, loc, "boolean"); + case "BreakStatement": + return convertBreakStatement(node, loc); + case "CallExpression": + return convertCallExpression(node, loc); + case "CallSignature": + return convertCallSignature(node, loc); + case "CaseClause": + return convertCaseClause(node, loc); + case "CatchClause": + return convertCatchClause(node, loc); + case "ClassDeclaration": + case "ClassExpression": + return convertClass(node, kind, loc); + case "CommaListExpression": + return convertCommaListExpression(node, loc); + case "ComputedPropertyName": + return convertComputedPropertyName(node); + case "ConditionalExpression": + return convertConditionalExpression(node, loc); + case "ConditionalType": + return convertConditionalType(node, loc); + case "Constructor": + return convertConstructor(node, loc); + case "ConstructSignature": + return convertConstructSignature(node, loc); + case "ConstructorType": + return convertConstructorType(node, loc); + case "ContinueStatement": + return convertContinueStatement(node, loc); + case "DebuggerStatement": + return convertDebuggerStatement(loc); + case "Decorator": + return convertDecorator(node, loc); + case "DefaultClause": + return convertCaseClause(node, loc); + case "DeleteExpression": + return convertDeleteExpression(node, loc); + case "DoStatement": + return convertDoStatement(node, loc); + case "ElementAccessExpression": + return convertElementAccessExpression(node, loc); + case "EmptyStatement": + return convertEmptyStatement(loc); + case "EnumDeclaration": + return convertEnumDeclaration(node, loc); + case "EnumMember": + return convertEnumMember(node, loc); + case "ExportAssignment": + return convertExportAssignment(node, loc); + case "ExportDeclaration": + return convertExportDeclaration(node, loc); + case "ExportSpecifier": + return convertExportSpecifier(node, loc); + case "ExpressionStatement": + return convertExpressionStatement(node, loc); + case "ExpressionWithTypeArguments": + return convertExpressionWithTypeArguments(node, loc); + case "ExternalModuleReference": + return convertExternalModuleReference(node, loc); + case "FalseKeyword": + return convertFalseKeyword(loc); + case "NeverKeyword": + return convertKeywordTypeExpr(node, loc, "never"); + case "NumberKeyword": + return convertKeywordTypeExpr(node, loc, "number"); + case "NumericLiteral": + return convertNumericLiteral(node, loc); + case "ForStatement": + return convertForStatement(node, loc); + case "ForInStatement": + return convertForInStatement(node, loc); + case "ForOfStatement": + return convertForOfStatement(node, loc); + case "FunctionDeclaration": + return convertFunctionDeclaration(node, loc); + case "FunctionExpression": + return convertFunctionExpression(node, loc); + case "FunctionType": + return convertFunctionType(node, loc); + case "Identifier": + return convertIdentifier(node, loc); + case "IfStatement": + return convertIfStatement(node, loc); + case "ImportClause": + return convertImportClause(node, loc); + case "ImportDeclaration": + return convertImportDeclaration(node, loc); + case "ImportEqualsDeclaration": + return convertImportEqualsDeclaration(node, loc); + case "ImportKeyword": + return convertImportKeyword(loc); + case "ImportSpecifier": + return convertImportSpecifier(node, loc); + case "ImportType": + return convertImportType(node, loc); + case "IndexSignature": + return convertIndexSignature(node, loc); + case "IndexedAccessType": + return convertIndexedAccessType(node, loc); + case "InferType": + return convertInferType(node, loc); + case "InterfaceDeclaration": + return convertInterfaceDeclaration(node, loc); + case "IntersectionType": + return convertIntersectionType(node, loc); + case "JsxAttribute": + return convertJsxAttribute(node, loc); + case "JsxClosingElement": + return convertJsxClosingElement(node, loc); + case "JsxElement": + return convertJsxElement(node, loc); + case "JsxExpression": + return convertJsxExpression(node, loc); + case "JsxFragment": + return convertJsxFragment(node, loc); + case "JsxOpeningElement": + return convertJsxOpeningElement(node, loc); + case "JsxOpeningFragment": + return convertJsxOpeningFragment(node, loc); + case "JsxSelfClosingElement": + return convertJsxSelfClosingElement(node, loc); + case "JsxClosingFragment": + return convertJsxClosingFragment(node, loc); + case "JsxSpreadAttribute": + return convertJsxSpreadAttribute(node, loc); + case "JsxText": + case "JsxTextAllWhiteSpaces": + return convertJsxText(node, loc); + case "LabeledStatement": + return convertLabeledStatement(node, loc); + case "LiteralType": + return convertLiteralType(node, loc); + case "MappedType": + return convertMappedType(node, loc); + case "MetaProperty": + return convertMetaProperty(node, loc); + case "GetAccessor": + case "SetAccessor": + case "MethodDeclaration": + case "MethodSignature": + return convertMethodDeclaration(node, kind, loc); + case "ModuleDeclaration": + case "NamespaceDeclaration": + return convertNamespaceDeclaration(node, loc); + case "ModuleBlock": + return convertModuleBlock(node, loc); + case "NamespaceExportDeclaration": + return convertNamespaceExportDeclaration(node, loc); + case "NamespaceImport": + return convertNamespaceImport(node, loc); + case "NewExpression": + return convertNewExpression(node, loc); + case "NonNullExpression": + return convertNonNullExpression(node, loc); + case "NoSubstitutionTemplateLiteral": + return convertNoSubstitutionTemplateLiteral(node, loc); + case "NullKeyword": + return convertNullKeyword(loc); + case "ObjectBindingPattern": + return convertObjectBindingPattern(node, loc); + case "ObjectKeyword": + return convertKeywordTypeExpr(node, loc, "object"); + case "ObjectLiteralExpression": + return convertObjectLiteralExpression(node, loc); + case "OmittedExpression": + return convertOmittedExpression(); + case "OptionalType": + return convertOptionalType(node, loc); + case "Parameter": + return convertParameter(node, loc); + case "ParenthesizedExpression": + return convertParenthesizedExpression(node, loc); + case "ParenthesizedType": + return convertParenthesizedType(node, loc); + case "PostfixUnaryExpression": + return convertPostfixUnaryExpression(node, loc); + case "PrefixUnaryExpression": + return convertPrefixUnaryExpression(node, loc); + case "PropertyAccessExpression": + return convertPropertyAccessExpression(node, loc); + case "PropertyAssignment": + return convertPropertyAssignment(node, loc); + case "PropertyDeclaration": + case "PropertySignature": + return convertPropertyDeclaration(node, kind, loc); + case "RegularExpressionLiteral": + return convertRegularExpressionLiteral(loc); + case "RestType": + return convertRestType(node, loc); + case "QualifiedName": + return convertQualifiedName(node, loc); + case "ReturnStatement": + return convertReturnStatement(node, loc); + case "SemicolonClassElement": + return convertSemicolonClassElement(); + case "SourceFile": + return convertSourceFile(node, loc); + case "ShorthandPropertyAssignment": + return convertShorthandPropertyAssignment(node, loc); + case "SpreadAssignment": + case "SpreadElement": + case "SpreadElementExpression": + return convertSpreadElement(node, loc); + case "StringKeyword": + return convertKeywordTypeExpr(node, loc, "string"); + case "StringLiteral": + return convertStringLiteral(node, loc); + case "SuperKeyword": + return convertSuperKeyword(loc); + case "SwitchStatement": + return convertSwitchStatement(node, loc); + case "SymbolKeyword": + return convertKeywordTypeExpr(node, loc, "symbol"); + case "TaggedTemplateExpression": + return convertTaggedTemplateExpression(node, loc); + case "TemplateExpression": + return convertTemplateExpression(node, loc); + case "TemplateHead": + case "TemplateMiddle": + case "TemplateTail": + return convertTemplateElement(node, kind, loc); + case "ThisKeyword": + return convertThisKeyword(loc); + case "ThisType": + return convertKeywordTypeExpr(node, loc, "this"); + case "ThrowStatement": + return convertThrowStatement(node, loc); + case "TrueKeyword": + return convertTrueKeyword(loc); + case "TryStatement": + return convertTryStatement(node, loc); + case "TupleType": + return convertTupleType(node, loc); + case "TypeAliasDeclaration": + return convertTypeAliasDeclaration(node, loc); + case "TypeAssertionExpression": + return convertTypeAssertionExpression(node, loc); + case "TypeLiteral": + return convertTypeLiteral(node, loc); + case "TypeOfExpression": + return convertTypeOfExpression(node, loc); + case "TypeOperator": + return convertTypeOperator(node, loc); + case "TypeParameter": + return convertTypeParameter(node, loc); + case "TypePredicate": + return convertTypePredicate(node, loc); + case "TypeReference": + return convertTypeReference(node, loc); + case "TypeQuery": + return convertTypeQuery(node, loc); + case "UndefinedKeyword": + return convertKeywordTypeExpr(node, loc, "undefined"); + case "UnionType": + return convertUnionType(node, loc); + case "UnknownKeyword": + return convertKeywordTypeExpr(node, loc, "unknown"); + case "VariableDeclaration": + return convertVariableDeclaration(node, loc); + case "VariableDeclarationList": + return convertVariableDeclarationList(node, loc); + case "VariableStatement": + return convertVariableStatement(node, loc); + case "VoidExpression": + return convertVoidExpression(node, loc); + case "VoidKeyword": + return convertKeywordTypeExpr(node, loc, "void"); + case "WhileStatement": + return convertWhileStatement(node, loc); + case "WithStatement": + return convertWithStatement(node, loc); + case "YieldExpression": + return convertYieldExpression(node, loc); + default: + throw new ParseError( + "Unsupported TypeScript syntax " + kind, getSourceLocation(node).getStart()); + } + } + + /** + * Attaches type information from the JSON object to the given AST node, if applicable. This is + * called from {@link #convertNode}. + */ + private void attachStaticType(Node astNode, JsonObject json) { + if (astNode instanceof ITypedAstNode && json.has("$type")) { + ITypedAstNode typedAstNode = (ITypedAstNode) astNode; + int typeId = json.get("$type").getAsInt(); + typedAstNode.setStaticTypeId(typeId); + } + } + + /** Attaches a TypeScript compiler symbol to the given node, if any was provided. */ + private void attachSymbolInformation(INodeWithSymbol node, JsonObject json) { + if (json.has("$symbol")) { + int symbol = json.get("$symbol").getAsInt(); + node.setSymbol(symbol); + } + } + + /** Attaches call signatures and related symbol information to a call site. */ + private void attachResolvedSignature(InvokeExpression node, JsonObject json) { + if (json.has("$resolvedSignature")) { + int id = json.get("$resolvedSignature").getAsInt(); + node.setResolvedSignatureId(id); + } + if (json.has("$overloadIndex")) { + int id = json.get("$overloadIndex").getAsInt(); + node.setOverloadIndex(id); + } + attachSymbolInformation(node, json); + } + + /** + * Convert the given array of TypeScript AST nodes into a list of JavaScript AST nodes, skipping + * any {@code null} elements. + */ + private List convertNodes(Iterable nodes) throws ParseError { + return convertNodes(nodes, true); + } + + /** + * Convert the given array of TypeScript AST nodes into a list of JavaScript AST nodes, where + * {@code skipNull} indicates whether {@code null} elements should be skipped or not. + */ + @SuppressWarnings("unchecked") + private List convertNodes(Iterable nodes, boolean skipNull) + throws ParseError { + List res = new ArrayList(); + for (JsonElement elt : nodes) { + T converted = (T) convertNode(elt.getAsJsonObject()); + if (!skipNull || converted != null) res.add(converted); + } + return res; + } + + /** + * Converts the given child to an AST node of the given type or null. A ParseError is + * thrown if a different type of node was found. + * + *

      This is used to detect syntax errors that are not reported as syntax errors by the + * TypeScript parser. Usually they are reported as errors in a later compiler stage, which the + * extractor does not run. + * + *

      Returns null if the child is absent. + */ + @SuppressWarnings("unchecked") + private T tryConvertChild(JsonObject node, String prop, Class expectedType) + throws ParseError { + Node child = convertChild(node, prop); + if (child == null || expectedType.isInstance(child)) { + return (T) child; + } else { + throw new ParseError("Unsupported TypeScript syntax", getSourceLocation(node).getStart()); + } + } + + /** Convert the child node named {@code prop} of AST node {@code node}. */ + private T convertChild(JsonObject node, String prop) throws ParseError { + return convertChild(node, prop, null); + } + + /** + * Convert the child node named {@code prop} of AST node {@code node}, with {@code kind} as its + * default kind. + */ + @SuppressWarnings("unchecked") + private T convertChild(JsonObject node, String prop, String kind) + throws ParseError { + JsonElement child = node.get(prop); + if (child == null) return null; + return (T) convertNode(child.getAsJsonObject(), kind); + } + + /** Convert the child nodes named {@code prop} of AST node {@code node}. */ + private List convertChildren(JsonObject node, String prop) + throws ParseError { + return convertChildren(node, prop, true); + } + + /** Like convertChildren but returns an empty list if the property is missing. */ + private List convertChildrenNotNull(JsonObject node, String prop) + throws ParseError { + List nodes = convertChildren(node, prop, true); + if (nodes == null) { + return Collections.emptyList(); + } + return nodes; + } + + /** + * Convert the child nodes named {@code prop} of AST node {@code node}, where {@code skipNull} + * indicates whether or not to skip null children. + */ + private List convertChildren(JsonObject node, String prop, boolean skipNull) + throws ParseError { + JsonElement child = node.get(prop); + if (child == null) return null; + return convertNodes(child.getAsJsonArray(), skipNull); + } + + /* Converter methods for the individual TypeScript AST node types. */ + + private Node convertArrayBindingPattern(JsonObject array, SourceLocation loc) throws ParseError { + List elements = new ArrayList<>(); + for (JsonElement elt : array.get("elements").getAsJsonArray()) { + JsonObject element = (JsonObject) elt; + SourceLocation eltLoc = getSourceLocation(element); + Expression convertedElt = convertChild(element, "name"); + if (hasChild(element, "initializer")) + convertedElt = + new AssignmentPattern(eltLoc, "=", convertedElt, convertChild(element, "initializer")); + else if (hasChild(element, "dotDotDotToken")) + convertedElt = new RestElement(eltLoc, convertedElt); + elements.add(convertedElt); + } + return new ArrayPattern(loc, elements); + } + + private Node convertArrayLiteralExpression(JsonObject node, SourceLocation loc) + throws ParseError { + return new ArrayExpression(loc, convertChildren(node, "elements", false)); + } + + private Node convertArrayType(JsonObject node, SourceLocation loc) throws ParseError { + return new ArrayTypeExpr(loc, convertChildAsType(node, "elementType")); + } + + private Node convertArrowFunction(JsonObject node, SourceLocation loc) throws ParseError { + return new ArrowFunctionExpression( + loc, + convertParameters(node), + convertChild(node, "body"), + false, + hasModifier(node, "AsyncKeyword"), + convertChildrenNotNull(node, "typeParameters"), + convertParameterTypes(node), + convertChildAsType(node, "type")); + } + + private Node convertAsExpression(JsonObject node, SourceLocation loc) throws ParseError { + return new TypeAssertion( + loc, convertChild(node, "expression"), convertChildAsType(node, "type"), true); + } + + private Node convertAwaitExpression(JsonObject node, SourceLocation loc) throws ParseError { + return new AwaitExpression(loc, convertChild(node, "expression")); + } + + private Node convertBigIntLiteral(JsonObject node, SourceLocation loc) throws ParseError { + String text = node.get("text").getAsString(); + String value = text.substring(0, text.length() - 1); // Remove the 'n' suffix. + return new Literal(loc, TokenType.bigint, value); + } + + private Node convertBinaryExpression(JsonObject node, SourceLocation loc) throws ParseError { + Expression left = convertChild(node, "left"); + Expression right = convertChild(node, "right"); + JsonObject operatorToken = node.get("operatorToken").getAsJsonObject(); + String operator = getSourceLocation(operatorToken).getSource(); + switch (operator) { + case ",": + List expressions = new ArrayList(); + if (left instanceof SequenceExpression) + expressions.addAll(((SequenceExpression) left).getExpressions()); + else expressions.add(left); + if (right instanceof SequenceExpression) + expressions.addAll(((SequenceExpression) right).getExpressions()); + else expressions.add(right); + return new SequenceExpression(loc, expressions); + + case "||": + case "&&": + return new LogicalExpression(loc, operator, left, right); + + case "=": + left = + convertLValue(left); // For plain assignments, the lhs can be a destructuring pattern. + return new AssignmentExpression(loc, operator, left, right); + + case "+=": + case "-=": + case "*=": + case "**=": + case "/=": + case "%=": + case "^=": + case "&=": + case "|=": + case ">>=": + case "<<=": + case ">>>=": + return new AssignmentExpression(loc, operator, convertLValue(left), right); + + default: + return new BinaryExpression(loc, operator, left, right); + } + } + + private Node convertBlock(JsonObject node, SourceLocation loc) throws ParseError { + return new BlockStatement(loc, convertChildren(node, "statements")); + } + + private Node convertBreakStatement(JsonObject node, SourceLocation loc) throws ParseError { + return new BreakStatement(loc, convertChild(node, "label")); + } + + private Node convertCallExpression(JsonObject node, SourceLocation loc) throws ParseError { + List arguments = convertChildren(node, "arguments"); + if (arguments.size() == 1 && hasKind(node.get("expression"), "ImportKeyword")) { + return new DynamicImport(loc, arguments.get(0)); + } + Expression callee = convertChild(node, "expression"); + List typeArguments = convertChildrenAsTypes(node, "typeArguments"); + CallExpression call = new CallExpression(loc, callee, typeArguments, arguments, false, false); + attachResolvedSignature(call, node); + return call; + } + + private MethodDefinition convertCallSignature(JsonObject node, SourceLocation loc) + throws ParseError { + FunctionExpression function = convertImplicitFunction(node, loc); + int flags = getMemberModifierKeywords(node) | DeclarationFlags.abstract_; + return new MethodDefinition(loc, flags, Kind.FUNCTION_CALL_SIGNATURE, null, function); + } + + private Node convertCaseClause(JsonObject node, SourceLocation loc) throws ParseError { + return convertDefaultClause(node, loc); + } + + private Node convertCatchClause(JsonObject node, SourceLocation loc) throws ParseError { + IPattern pattern = null; + JsonElement variableDecl = node.get("variableDeclaration"); + if (variableDecl != null) pattern = convertChild(variableDecl.getAsJsonObject(), "name"); + return new CatchClause(loc, pattern, null, convertChild(node, "block")); + } + + private List convertSuperInterfaceClause(JsonArray supers) throws ParseError { + List result = new ArrayList<>(); + for (JsonElement elt : supers) { + JsonObject superType = elt.getAsJsonObject(); + ITypeExpression objectType = convertChildAsType(superType, "expression"); + if (objectType == null) continue; + List typeArguments = convertChildrenAsTypes(superType, "typeArguments"); + if (typeArguments.isEmpty()) { + result.add(objectType); + } else { + result.add(new GenericTypeExpr(getSourceLocation(superType), objectType, typeArguments)); + } + } + return result; + } + + private Node convertClass(JsonObject node, String kind, SourceLocation loc) throws ParseError { + Identifier id = convertChild(node, "name"); + List typeParameters = convertChildrenNotNull(node, "typeParameters"); + Expression superClass = null; + List superInterfaces = null; + int afterHead = id == null ? loc.getStart().getOffset() + 5 : id.getLoc().getEnd().getOffset(); + for (JsonElement elt : getChildIterable(node, "heritageClauses")) { + JsonObject heritageClause = elt.getAsJsonObject(); + JsonArray supers = heritageClause.get("types").getAsJsonArray(); + if (heritageClause.get("token").getAsInt() == syntaxKindExtends) { + if (supers.size() > 0) { + superClass = (Expression) convertNode(supers.get(0).getAsJsonObject()); + } + } else { + superInterfaces = convertSuperInterfaceClause(supers); + } + afterHead = heritageClause.get("$end").getAsInt(); + } + if (superInterfaces == null) { + superInterfaces = new ArrayList<>(); + } + String skip = + source.substring(loc.getStart().getOffset(), afterHead) + matchWhitespace(afterHead); + SourceLocation bodyLoc = new SourceLocation(loc.getSource(), loc.getStart(), loc.getEnd()); + advance(bodyLoc, skip); + ClassBody body = new ClassBody(bodyLoc, convertChildren(node, "members")); + if ("ClassExpression".equals(kind)) { + ClassExpression classExpr = + new ClassExpression(loc, id, typeParameters, superClass, superInterfaces, body); + attachSymbolInformation(classExpr.getClassDef(), node); + return classExpr; + } + boolean hasDeclareKeyword = hasModifier(node, "DeclareKeyword"); + boolean hasAbstractKeyword = hasModifier(node, "AbstractKeyword"); + ClassDeclaration classDecl = + new ClassDeclaration( + loc, + id, + typeParameters, + superClass, + superInterfaces, + body, + hasDeclareKeyword, + hasAbstractKeyword); + attachSymbolInformation(classDecl.getClassDef(), node); + if (node.has("decorators")) { + classDecl.addDecorators(convertChildren(node, "decorators")); + advanceUntilAfter(loc, classDecl.getDecorators()); + } + return fixExports(loc, classDecl); + } + + private Node convertCommaListExpression(JsonObject node, SourceLocation loc) throws ParseError { + return new SequenceExpression(loc, convertChildren(node, "elements")); + } + + private Node convertComputedPropertyName(JsonObject node) throws ParseError { + return convertChild(node, "expression"); + } + + private Node convertConditionalExpression(JsonObject node, SourceLocation loc) throws ParseError { + return new ConditionalExpression( + loc, + convertChild(node, "condition"), + convertChild(node, "whenTrue"), + convertChild(node, "whenFalse")); + } + + private Node convertConditionalType(JsonObject node, SourceLocation loc) throws ParseError { + return new ConditionalTypeExpr( + loc, + convertChild(node, "checkType"), + convertChild(node, "extendsType"), + convertChild(node, "trueType"), + convertChild(node, "falseType")); + } + + private SourceLocation getSourceRange(Position from, Position to) { + return new SourceLocation(source.substring(from.getOffset(), to.getOffset()), from, to); + } + + private DecoratorList makeDecoratorList(JsonElement decorators) throws ParseError { + if (!(decorators instanceof JsonArray)) return null; + JsonArray array = decorators.getAsJsonArray(); + SourceLocation firstLoc = null, lastLoc = null; + List list = new ArrayList<>(); + for (JsonElement decoratorElm : array) { + JsonObject decorator = decoratorElm.getAsJsonObject(); + if (hasKind(decorator, "Decorator")) { + SourceLocation location = getSourceLocation(decorator); + list.add(convertDecorator(decorator, location)); + if (firstLoc == null) { + firstLoc = location; + } + lastLoc = location; + } + } + if (firstLoc == null) return null; + return new DecoratorList(getSourceRange(firstLoc.getStart(), lastLoc.getEnd()), list); + } + + private List convertParameterDecorators(JsonObject function) throws ParseError { + List decoratorLists = new ArrayList<>(); + for (JsonElement parameter : getProperParameters(function)) { + decoratorLists.add(makeDecoratorList(parameter.getAsJsonObject().get("decorators"))); + } + return decoratorLists; + } + + private Node convertConstructor(JsonObject node, SourceLocation loc) throws ParseError { + int flags = getMemberModifierKeywords(node); + boolean isComputed = hasComputedName(node); + boolean isStatic = DeclarationFlags.isStatic(flags); + if (isComputed) { + flags |= DeclarationFlags.computed; + } + // for some reason, the TypeScript compiler treats static methods named "constructor" + // and methods with computed name "constructor" as constructors, even though they aren't + MethodDefinition.Kind methodKind = isStatic || isComputed ? Kind.METHOD : Kind.CONSTRUCTOR; + Expression key; + if (isComputed) key = convertChild((JsonObject) node.get("name"), "expression"); + else key = new Identifier(loc, "constructor"); + List params = convertParameters(node); + List paramTypes = convertParameterTypes(node); + List paramDecorators = convertParameterDecorators(node); + FunctionExpression value = + new FunctionExpression( + loc, + null, + params, + convertChild(node, "body"), + false, + false, + Collections.emptyList(), + paramTypes, + paramDecorators, + null, + null); + attachSymbolInformation(value, node); + List parameterFields = convertParameterFields(node); + return new MethodDefinition(loc, flags, methodKind, key, value, parameterFields); + } + + private MethodDefinition convertConstructSignature(JsonObject node, SourceLocation loc) + throws ParseError { + FunctionExpression function = convertImplicitFunction(node, loc); + int flags = getMemberModifierKeywords(node) | DeclarationFlags.abstract_; + return new MethodDefinition(loc, flags, Kind.CONSTRUCTOR_CALL_SIGNATURE, null, function); + } + + private Node convertConstructorType(JsonObject node, SourceLocation loc) throws ParseError { + return new FunctionTypeExpr(loc, convertImplicitFunction(node, loc), true); + } + + private Node convertContinueStatement(JsonObject node, SourceLocation loc) throws ParseError { + return new ContinueStatement(loc, convertChild(node, "label")); + } + + private Node convertDebuggerStatement(SourceLocation loc) { + return new DebuggerStatement(loc); + } + + private Decorator convertDecorator(JsonObject node, SourceLocation loc) throws ParseError { + return new Decorator(loc, convertChild(node, "expression")); + } + + private Node convertDefaultClause(JsonObject node, SourceLocation loc) throws ParseError { + return new SwitchCase( + loc, convertChild(node, "expression"), convertChildren(node, "statements")); + } + + private Node convertDeleteExpression(JsonObject node, SourceLocation loc) throws ParseError { + return new UnaryExpression(loc, "delete", convertChild(node, "expression"), true); + } + + private Node convertDoStatement(JsonObject node, SourceLocation loc) throws ParseError { + return new DoWhileStatement( + loc, convertChild(node, "expression"), convertChild(node, "statement")); + } + + private Node convertElementAccessExpression(JsonObject node, SourceLocation loc) + throws ParseError { + Expression object = convertChild(node, "expression"); + Expression property = convertChild(node, "argumentExpression"); + return new MemberExpression(loc, object, property, true, false, false); + } + + private Node convertEmptyStatement(SourceLocation loc) { + return new EmptyStatement(loc); + } + + private Node convertEnumDeclaration(JsonObject node, SourceLocation loc) throws ParseError { + EnumDeclaration enumDeclaration = + new EnumDeclaration( + loc, + hasModifier(node, "ConstKeyword"), + hasModifier(node, "DeclareKeyword"), + convertChildrenNotNull(node, "decorators"), + convertChild(node, "name"), + convertChildren(node, "members")); + attachSymbolInformation(enumDeclaration, node); + advanceUntilAfter(loc, enumDeclaration.getDecorators()); + return fixExports(loc, enumDeclaration); + } + + /** + * Converts a TypeScript Identifier or StringLiteral node to an Identifier AST node, or {@code + * null} if the given node is not of the expected kind. + */ + private Identifier convertNodeAsIdentifier(JsonObject node) throws ParseError { + SourceLocation loc = getSourceLocation(node); + if (isIdentifier(node)) { + return convertIdentifier(node, loc); + } else if (hasKind(node, "StringLiteral")) { + return new Identifier(loc, node.get("text").getAsString()); + } else { + return null; + } + } + + private Node convertEnumMember(JsonObject node, SourceLocation loc) throws ParseError { + Identifier name = convertNodeAsIdentifier(node.get("name").getAsJsonObject()); + if (name == null) return null; + EnumMember member = new EnumMember(loc, name, convertChild(node, "initializer")); + attachSymbolInformation(member, node); + return member; + } + + private Node convertExportAssignment(JsonObject node, SourceLocation loc) throws ParseError { + if (hasChild(node, "isExportEquals") && node.get("isExportEquals").getAsBoolean()) + return new ExportWholeDeclaration(loc, convertChild(node, "expression")); + return new ExportDefaultDeclaration(loc, convertChild(node, "expression")); + } + + private Node convertExportDeclaration(JsonObject node, SourceLocation loc) throws ParseError { + Literal source = tryConvertChild(node, "moduleSpecifier", Literal.class); + if (hasChild(node, "exportClause")) { + return new ExportNamedDeclaration( + loc, + null, + convertChildren(node.get("exportClause").getAsJsonObject(), "elements"), + source); + } else { + return new ExportAllDeclaration(loc, source); + } + } + + private Node convertExportSpecifier(JsonObject node, SourceLocation loc) throws ParseError { + return new ExportSpecifier( + loc, + convertChild(node, hasChild(node, "propertyName") ? "propertyName" : "name"), + convertChild(node, "name")); + } + + private Node convertExpressionStatement(JsonObject node, SourceLocation loc) throws ParseError { + Expression expression = convertChild(node, "expression"); + return new ExpressionStatement(loc, expression); + } + + private Node convertExpressionWithTypeArguments(JsonObject node, SourceLocation loc) + throws ParseError { + Expression expression = convertChild(node, "expression"); + List typeArguments = convertChildrenAsTypes(node, "typeArguments"); + if (typeArguments.isEmpty()) return expression; + return new ExpressionWithTypeArguments(loc, expression, typeArguments); + } + + private Node convertExternalModuleReference(JsonObject node, SourceLocation loc) + throws ParseError { + return new ExternalModuleReference(loc, convertChild(node, "expression")); + } + + private Node convertFalseKeyword(SourceLocation loc) { + return new Literal(loc, TokenType._false, false); + } + + private Node convertNumericLiteral(JsonObject node, SourceLocation loc) + throws NumberFormatException { + return new Literal(loc, TokenType.num, Double.valueOf(node.get("text").getAsString())); + } + + private Node convertForStatement(JsonObject node, SourceLocation loc) throws ParseError { + return new ForStatement( + loc, + convertChild(node, "initializer"), + convertChild(node, "condition"), + convertChild(node, "incrementor"), + convertChild(node, "statement")); + } + + private Node convertForInStatement(JsonObject node, SourceLocation loc) throws ParseError { + Node initializer = convertChild(node, "initializer"); + if (initializer instanceof Expression) initializer = convertLValue((Expression) initializer); + return new ForInStatement( + loc, initializer, convertChild(node, "expression"), convertChild(node, "statement"), false); + } + + private Node convertForOfStatement(JsonObject node, SourceLocation loc) throws ParseError { + Node initializer = convertChild(node, "initializer"); + if (initializer instanceof Expression) initializer = convertLValue((Expression) initializer); + return new ForOfStatement( + loc, initializer, convertChild(node, "expression"), convertChild(node, "statement")); + } + + private Node convertFunctionDeclaration(JsonObject node, SourceLocation loc) throws ParseError { + List params = convertParameters(node); + Identifier fnId = convertChild(node, "name", "Identifier"); + BlockStatement fnbody = convertChild(node, "body"); + boolean generator = hasChild(node, "asteriskToken"); + boolean async = hasModifier(node, "AsyncKeyword"); + boolean hasDeclareKeyword = hasModifier(node, "DeclareKeyword"); + List paramTypes = convertParameterTypes(node); + List typeParameters = convertChildrenNotNull(node, "typeParameters"); + ITypeExpression returnType = convertChildAsType(node, "type"); + ITypeExpression thisParam = convertThisParameterType(node); + FunctionDeclaration function = + new FunctionDeclaration( + loc, + fnId, + params, + fnbody, + generator, + async, + hasDeclareKeyword, + typeParameters, + paramTypes, + returnType, + thisParam); + attachSymbolInformation(function, node); + return fixExports(loc, function); + } + + private Node convertFunctionExpression(JsonObject node, SourceLocation loc) throws ParseError { + Identifier fnId = convertChild(node, "name", "Identifier"); + List params = convertParameters(node); + BlockStatement fnbody = convertChild(node, "body"); + boolean generator = hasChild(node, "asteriskToken"); + boolean async = hasModifier(node, "AsyncKeyword"); + List paramTypes = convertParameterTypes(node); + List paramDecorators = convertParameterDecorators(node); + ITypeExpression returnType = convertChildAsType(node, "type"); + ITypeExpression thisParam = convertThisParameterType(node); + return new FunctionExpression( + loc, + fnId, + params, + fnbody, + generator, + async, + convertChildrenNotNull(node, "typeParameters"), + paramTypes, + paramDecorators, + returnType, + thisParam); + } + + private Node convertFunctionType(JsonObject node, SourceLocation loc) throws ParseError { + return new FunctionTypeExpr(loc, convertImplicitFunction(node, loc), false); + } + + /** Gets the original text out of an Identifier's "escapedText" field. */ + private String unescapeLeadingUnderscores(String text) { + // The TypeScript compiler inserts an additional underscore in front of + // identifiers that begin with two underscores. + if (text.startsWith("___")) { + return text.substring(1); + } else { + return text; + } + } + + /** Returns the contents of the given identifier as a string. */ + private String getIdentifierText(JsonObject identifierNode) { + if (identifierNode.has("text")) return identifierNode.get("text").getAsString(); + else return unescapeLeadingUnderscores(identifierNode.get("escapedText").getAsString()); + } + + private Identifier convertIdentifier(JsonObject node, SourceLocation loc) { + Identifier id = new Identifier(loc, getIdentifierText(node)); + attachSymbolInformation(id, node); + return id; + } + + private Node convertKeywordTypeExpr(JsonObject node, SourceLocation loc, String text) { + return new KeywordTypeExpr(loc, text); + } + + private Node convertUnionType(JsonObject node, SourceLocation loc) throws ParseError { + return new UnionTypeExpr(loc, convertChildrenAsTypes(node, "types")); + } + + private Node convertIfStatement(JsonObject node, SourceLocation loc) throws ParseError { + return new IfStatement( + loc, + convertChild(node, "expression"), + convertChild(node, "thenStatement"), + convertChild(node, "elseStatement")); + } + + private Node convertImportClause(JsonObject node, SourceLocation loc) throws ParseError { + return new ImportDefaultSpecifier(loc, convertChild(node, "name")); + } + + private Node convertImportDeclaration(JsonObject node, SourceLocation loc) throws ParseError { + Literal src = tryConvertChild(node, "moduleSpecifier", Literal.class); + List specifiers = new ArrayList<>(); + if (hasChild(node, "importClause")) { + JsonObject importClause = node.get("importClause").getAsJsonObject(); + if (hasChild(importClause, "name")) { + specifiers.add(convertChild(node, "importClause")); + } + if (hasChild(importClause, "namedBindings")) { + JsonObject namedBindings = importClause.get("namedBindings").getAsJsonObject(); + if (hasKind(namedBindings, "NamespaceImport")) { + specifiers.add(convertChild(importClause, "namedBindings")); + } else { + specifiers.addAll(convertChildren(namedBindings, "elements")); + } + } + } + return new ImportDeclaration(loc, specifiers, src); + } + + private Node convertImportEqualsDeclaration(JsonObject node, SourceLocation loc) + throws ParseError { + return fixExports( + loc, + new ImportWholeDeclaration( + loc, convertChild(node, "name"), convertChild(node, "moduleReference"))); + } + + private Node convertImportKeyword(SourceLocation loc) { + return new Identifier(loc, "import"); + } + + private Node convertImportSpecifier(JsonObject node, SourceLocation loc) throws ParseError { + boolean hasImported = hasChild(node, "propertyName"); + Identifier imported = convertChild(node, hasImported ? "propertyName" : "name"); + Identifier local = convertChild(node, "name"); + return new ImportSpecifier(loc, imported, local); + } + + private Node convertImportType(JsonObject node, SourceLocation loc) throws ParseError { + // This is a type such as `import("./foo").bar.Baz`. + // + // The TypeScript AST represents import types as the root of a qualified name, + // whereas we represent them as the leftmost qualifier. + // + // So in our AST, ImportTypeExpr just represents `import("./foo")`, and `.bar.Baz` + // is represented by nested MemberExpr nodes. + // + // Additionally, an import type can be prefixed by `typeof`, such as `typeof import("foo")`. + // We convert these to TypeofTypeExpr. + + // Get the source range of the `import(path)` part. + Position importStart = loc.getStart(); + Position importEnd = loc.getEnd(); + boolean isTypeof = false; + if (node.has("isTypeOf") && node.get("isTypeOf").getAsBoolean() == true) { + isTypeof = true; + Matcher m = TYPEOF_START.matcher(loc.getSource()); + if (m.find()) { + importStart = advance(importStart, m.group(0)); + } + } + // Find the ending parenthesis in `import(path)` by skipping whitespace after `path`. + ITypeExpression path = convertChild(node, "argument"); + String endSrc = + loc.getSource().substring(path.getLoc().getEnd().getOffset() - loc.getStart().getOffset()); + Matcher m = WHITESPACE_END_PAREN.matcher(endSrc); + if (m.find()) { + importEnd = advance(path.getLoc().getEnd(), m.group(0)); + } + SourceLocation importLoc = getSourceRange(importStart, importEnd); + ImportTypeExpr imprt = new ImportTypeExpr(importLoc, path); + + ITypeExpression typeName = buildQualifiedTypeAccess(imprt, (JsonObject) node.get("qualifier")); + if (isTypeof) { + return new TypeofTypeExpr(loc, typeName); + } + + List typeArguments = convertChildrenAsTypes(node, "typeArguments"); + if (!typeArguments.isEmpty()) { + return new GenericTypeExpr(loc, typeName, typeArguments); + } + return (Node) typeName; + } + + /** + * Converts the given JSON to a qualified name with `root` as the base. + * + *

      For example, `a.b.c` is converted to the AST corresponding to `root.a.b.c`. + */ + private ITypeExpression buildQualifiedTypeAccess(ITypeExpression root, JsonObject node) + throws ParseError { + if (node == null) { + return root; + } + String kind = getKind(node); + ITypeExpression base; + Expression name; + if (kind == null || kind.equals("Identifier")) { + base = root; + name = convertIdentifier(node, getSourceLocation(node)); + } else if (kind.equals("QualifiedName")) { + base = buildQualifiedTypeAccess(root, (JsonObject) node.get("left")); + name = convertChild(node, "right"); + } else { + throw new ParseError("Unsupported syntax in import type", getSourceLocation(node).getStart()); + } + MemberExpression member = + new MemberExpression(getSourceLocation(node), (Expression) base, name, false, false, false); + attachSymbolInformation(member, node); + return member; + } + + private Node convertIndexSignature(JsonObject node, SourceLocation loc) throws ParseError { + FunctionExpression function = convertImplicitFunction(node, loc); + int flags = getMemberModifierKeywords(node) | DeclarationFlags.abstract_; + return new MethodDefinition(loc, flags, Kind.INDEX_SIGNATURE, null, function); + } + + private Node convertIndexedAccessType(JsonObject node, SourceLocation loc) throws ParseError { + return new IndexedAccessTypeExpr( + loc, convertChildAsType(node, "objectType"), convertChildAsType(node, "indexType")); + } + + private Node convertInferType(JsonObject node, SourceLocation loc) throws ParseError { + return new InferTypeExpr(loc, convertChild(node, "typeParameter")); + } + + private Node convertInterfaceDeclaration(JsonObject node, SourceLocation loc) throws ParseError { + Identifier name = convertChild(node, "name"); + List typeParameters = convertChildrenNotNull(node, "typeParameters"); + List> members = convertChildren(node, "members"); + List superInterfaces = null; + for (JsonElement elt : getChildIterable(node, "heritageClauses")) { + JsonObject heritageClause = elt.getAsJsonObject(); + if (heritageClause.get("token").getAsInt() == syntaxKindExtends) { + superInterfaces = convertSuperInterfaceClause(heritageClause.get("types").getAsJsonArray()); + break; + } + } + if (superInterfaces == null) { + superInterfaces = new ArrayList<>(); + } + InterfaceDeclaration iface = + new InterfaceDeclaration(loc, name, typeParameters, superInterfaces, members); + attachSymbolInformation(iface, node); + return fixExports(loc, iface); + } + + private Node convertIntersectionType(JsonObject node, SourceLocation loc) throws ParseError { + return new IntersectionTypeExpr(loc, convertChildrenAsTypes(node, "types")); + } + + private Node convertJsxAttribute(JsonObject node, SourceLocation loc) throws ParseError { + return new JSXAttribute( + loc, convertJSXName(convertChild(node, "name")), convertChild(node, "initializer")); + } + + private Node convertJsxClosingElement(JsonObject node, SourceLocation loc) throws ParseError { + return new JSXClosingElement(loc, convertJSXName(convertChild(node, "tagName"))); + } + + private Node convertJsxElement(JsonObject node, SourceLocation loc) throws ParseError { + return new JSXElement( + loc, + convertChild(node, "openingElement"), + convertChildren(node, "children"), + convertChild(node, "closingElement")); + } + + private Node convertJsxExpression(JsonObject node, SourceLocation loc) throws ParseError { + if (hasChild(node, "expression")) + return new JSXExpressionContainer(loc, convertChild(node, "expression")); + return new JSXExpressionContainer(loc, new JSXEmptyExpression(loc)); + } + + private Node convertJsxFragment(JsonObject node, SourceLocation loc) throws ParseError { + return new JSXElement( + loc, + convertChild(node, "openingFragment"), + convertChildren(node, "children"), + convertChild(node, "closingFragment")); + } + + private Node convertJsxOpeningFragment(JsonObject node, SourceLocation loc) { + return new JSXOpeningElement(loc, null, Collections.emptyList(), false); + } + + private Node convertJsxClosingFragment(JsonObject node, SourceLocation loc) { + return new JSXClosingElement(loc, null); + } + + private List convertJsxAttributes(JsonObject node) throws ParseError { + JsonElement attributes = node.get("attributes"); + List convertedAttributes; + if (attributes.isJsonArray()) { + convertedAttributes = convertNodes(attributes.getAsJsonArray()); + } else { + convertedAttributes = convertChildren(attributes.getAsJsonObject(), "properties"); + } + return convertedAttributes; + } + + private Node convertJsxOpeningElement(JsonObject node, SourceLocation loc) throws ParseError { + List convertedAttributes = convertJsxAttributes(node); + return new JSXOpeningElement( + loc, + convertJSXName(convertChild(node, "tagName")), + convertedAttributes, + hasChild(node, "selfClosing")); + } + + private Node convertJsxSelfClosingElement(JsonObject node, SourceLocation loc) throws ParseError { + List convertedAttributes = convertJsxAttributes(node); + JSXOpeningElement opening = + new JSXOpeningElement( + loc, convertJSXName(convertChild(node, "tagName")), convertedAttributes, true); + return new JSXElement(loc, opening, new ArrayList<>(), null); + } + + private Node convertJsxSpreadAttribute(JsonObject node, SourceLocation loc) throws ParseError { + return new JSXSpreadAttribute(loc, convertChild(node, "expression")); + } + + private Node convertJsxText(JsonObject node, SourceLocation loc) { + String text; + if (hasChild(node, "text")) text = node.get("text").getAsString(); + else text = ""; + return new Literal(loc, TokenType.string, text); + } + + private Node convertLabeledStatement(JsonObject node, SourceLocation loc) throws ParseError { + return new LabeledStatement(loc, convertChild(node, "label"), convertChild(node, "statement")); + } + + private Node convertLiteralType(JsonObject node, SourceLocation loc) throws ParseError { + return convertChild(node, "literal"); + } + + private Node convertMappedType(JsonObject node, SourceLocation loc) throws ParseError { + return new MappedTypeExpr( + loc, convertChild(node, "typeParameter"), convertChildAsType(node, "type")); + } + + private Node convertMetaProperty(JsonObject node, SourceLocation loc) throws ParseError { + Position metaStart = loc.getStart(); + Position metaEnd = + new Position(metaStart.getLine(), metaStart.getColumn() + 3, metaStart.getOffset() + 3); + SourceLocation metaLoc = new SourceLocation("new", metaStart, metaEnd); + Identifier meta = new Identifier(metaLoc, "new"); + return new MetaProperty(loc, meta, convertChild(node, "name")); + } + + private Node convertMethodDeclaration(JsonObject node, String kind, SourceLocation loc) + throws ParseError { + int flags = getMemberModifierKeywords(node); + if (hasComputedName(node)) { + flags |= DeclarationFlags.computed; + } + if (kind.equals("MethodSignature")) { + flags |= DeclarationFlags.abstract_; + } + MethodDefinition.Kind methodKind; + if ("GetAccessor".equals(kind)) methodKind = Kind.GET; + else if ("SetAccessor".equals(kind)) methodKind = Kind.SET; + else methodKind = Kind.METHOD; + FunctionExpression method = convertImplicitFunction(node, loc); + MethodDefinition methodDefinition = + new MethodDefinition(loc, flags, methodKind, convertChild(node, "name"), method); + if (node.has("decorators")) { + methodDefinition.addDecorators(convertChildren(node, "decorators")); + advanceUntilAfter(loc, methodDefinition.getDecorators()); + } + return methodDefinition; + } + + private FunctionExpression convertImplicitFunction(JsonObject node, SourceLocation loc) + throws ParseError { + ITypeExpression returnType = convertChildAsType(node, "type"); + List paramTypes = convertParameterTypes(node); + List paramDecorators = convertParameterDecorators(node); + List typeParameters = convertChildrenNotNull(node, "typeParameters"); + FunctionExpression method = + new FunctionExpression( + loc, + null, + convertParameters(node), + convertChild(node, "body"), + hasChild(node, "asteriskToken"), + hasModifier(node, "AsyncKeyword"), + typeParameters, + paramTypes, + paramDecorators, + returnType, + null); + attachSymbolInformation(method, node); + return method; + } + + private Node convertNamespaceDeclaration(JsonObject node, SourceLocation loc) throws ParseError { + Node nameNode = convertChild(node, "name"); + List body; + Statement b = convertChild(node, "body"); + if (b instanceof BlockStatement) { + body = ((BlockStatement) b).getBody(); + } else { + body = new ArrayList<>(); + body.add(b); + } + if (nameNode instanceof Literal) { + // Declaration of form: declare module "X" {...} + return new ExternalModuleDeclaration(loc, (Literal) nameNode, body); + } + if (hasFlag(node, "GlobalAugmentation")) { + // Declaration of form: declare global {...} + return new GlobalAugmentationDeclaration(loc, body); + } + Identifier name = (Identifier) nameNode; + boolean isInstantiated = false; + for (Statement stmt : body) { + isInstantiated = isInstantiated || isInstantiatingNamespaceMember(stmt); + } + boolean hasDeclareKeyword = hasModifier(node, "DeclareKeyword"); + NamespaceDeclaration decl = + new NamespaceDeclaration(loc, name, body, isInstantiated, hasDeclareKeyword); + attachSymbolInformation(decl, node); + if (hasFlag(node, "NestedNamespace")) { + // In a nested namespace declaration `namespace A.B`, the nested namespace `B` + // is implicitly exported. + return new ExportNamedDeclaration(loc, decl, new ArrayList<>(), null); + } else { + return fixExports(loc, decl); + } + } + + private boolean isInstantiatingNamespaceMember(Statement node) { + if (node instanceof ExportNamedDeclaration) { + // Ignore 'export' modifiers. + return isInstantiatingNamespaceMember(((ExportNamedDeclaration) node).getDeclaration()); + } + if (node instanceof NamespaceDeclaration) { + return ((NamespaceDeclaration) node).isInstantiated(); + } + if (node instanceof InterfaceDeclaration) { + return false; + } + if (node instanceof TypeAliasDeclaration) { + return false; + } + return true; + } + + private Node convertModuleBlock(JsonObject node, SourceLocation loc) throws ParseError { + return convertBlock(node, loc); + } + + private Node convertNamespaceExportDeclaration(JsonObject node, SourceLocation loc) + throws ParseError { + return new ExportAsNamespaceDeclaration(loc, convertChild(node, "name")); + } + + private Node convertNamespaceImport(JsonObject node, SourceLocation loc) throws ParseError { + return new ImportNamespaceSpecifier(loc, convertChild(node, "name")); + } + + private Node convertNewExpression(JsonObject node, SourceLocation loc) throws ParseError { + List arguments; + if (hasChild(node, "arguments")) arguments = convertChildren(node, "arguments"); + else arguments = new ArrayList<>(); + List typeArguments = convertChildrenAsTypes(node, "typeArguments"); + NewExpression result = + new NewExpression(loc, convertChild(node, "expression"), typeArguments, arguments); + attachResolvedSignature(result, node); + return result; + } + + private Node convertNonNullExpression(JsonObject node, SourceLocation loc) throws ParseError { + return new NonNullAssertion(loc, convertChild(node, "expression")); + } + + private Node convertNoSubstitutionTemplateLiteral(JsonObject node, SourceLocation loc) { + List quasis = new ArrayList<>(); + TemplateElement elm = + new TemplateElement( + loc, + node.get("text").getAsString(), + loc.getSource().substring(1, loc.getSource().length() - 1), + true); + quasis.add(elm); + attachStaticType(elm, node); + return new TemplateLiteral(loc, new ArrayList<>(), quasis); + } + + private Node convertNullKeyword(SourceLocation loc) { + return new Literal(loc, TokenType._null, null); + } + + private Node convertObjectBindingPattern(JsonObject node, SourceLocation loc) throws ParseError { + List properties = new ArrayList<>(); + for (JsonElement elt : node.get("elements").getAsJsonArray()) { + JsonObject element = elt.getAsJsonObject(); + SourceLocation eltLoc = getSourceLocation(element); + Expression propKey = + hasChild(element, "propertyName") + ? convertChild(element, "propertyName") + : convertChild(element, "name"); + Expression propVal; + if (hasChild(element, "dotDotDotToken")) { + propVal = new RestElement(eltLoc, propKey); + } else if (hasChild(element, "initializer")) { + propVal = + new AssignmentPattern( + eltLoc, "=", convertChild(element, "name"), convertChild(element, "initializer")); + } else { + propVal = convertChild(element, "name"); + } + properties.add( + new Property( + eltLoc, propKey, propVal, "init", hasComputedName(element, "propertyName"), false)); + } + return new ObjectPattern(loc, properties); + } + + private Node convertObjectLiteralExpression(JsonObject node, SourceLocation loc) + throws ParseError { + List properties; + properties = new ArrayList(); + for (INode e : convertChildren(node, "properties")) { + if (e instanceof SpreadElement) { + properties.add( + new Property( + e.getLoc(), null, (Expression) e, Property.Kind.INIT.name(), false, false)); + } else if (e instanceof MethodDefinition) { + MethodDefinition md = (MethodDefinition) e; + Property.Kind kind = Property.Kind.INIT; + if (md.getKind() == Kind.GET) { + kind = Property.Kind.GET; + } else if (md.getKind() == Kind.SET) { + kind = Property.Kind.SET; + } + properties.add( + new Property( + e.getLoc(), md.getKey(), md.getValue(), kind.name(), md.isComputed(), true)); + } else { + properties.add((Property) e); + } + } + return new ObjectExpression(loc, properties); + } + + private Node convertOmittedExpression() { + return null; + } + + private Node convertOptionalType(JsonObject node, SourceLocation loc) throws ParseError { + return new OptionalTypeExpr(loc, convertChild(node, "type")); + } + + private ITypeExpression asType(Node node) { + return node instanceof ITypeExpression ? (ITypeExpression) node : null; + } + + private List convertChildrenAsTypes(JsonObject node, String child) + throws ParseError { + List result = new ArrayList<>(); + JsonElement children = node.get(child); + if (!(children instanceof JsonArray)) return result; + for (JsonElement childNode : children.getAsJsonArray()) { + ITypeExpression type = asType(convertNode(childNode.getAsJsonObject())); + if (type != null) result.add(type); + } + return result; + } + + private ITypeExpression convertChildAsType(JsonObject node, String child) throws ParseError { + return asType(convertChild(node, child)); + } + + /** True if the given node is an Identifier node. */ + private boolean isIdentifier(JsonElement node) { + if (node == null) return false; + JsonObject object = node.getAsJsonObject(); + if (object == null) return false; + String kind = getKind(object); + return kind == null || kind.equals("Identifier"); + } + + /** + * Returns true if this is the JSON object for the special "this" parameter. + * + *

      It should be given the JSON object of kind "Parameter". + */ + private boolean isThisParameter(JsonElement parameter) { + JsonObject name = parameter.getAsJsonObject().get("name").getAsJsonObject(); + return isIdentifier(name) && getIdentifierText(name).equals("this"); + } + + /** + * Returns the parameters of the given function, omitting the special "this" parameter, which we + * do not consider to be a proper parameter. + */ + private Iterable getProperParameters(JsonObject function) { + if (!function.has("parameters")) return Collections.emptyList(); + JsonArray parameters = function.get("parameters").getAsJsonArray(); + if (parameters.size() > 0 && isThisParameter(parameters.get(0))) { + return CollectionUtil.skipIterable(parameters, 1); + } else { + return parameters; + } + } + + /** + * Returns the special "this" parameter of the given function, or {@code null} if the function + * does not declare a "this" parameter. + */ + private ITypeExpression convertThisParameterType(JsonObject function) throws ParseError { + if (!function.has("parameters")) return null; + JsonArray parameters = function.get("parameters").getAsJsonArray(); + if (parameters.size() > 0 && isThisParameter(parameters.get(0))) { + return convertChildAsType(parameters.get(0).getAsJsonObject(), "type"); + } else { + return null; + } + } + + private List convertParameters(JsonObject function) throws ParseError { + return convertNodes(getProperParameters(function), true); + } + + private List convertParameterTypes(JsonObject function) throws ParseError { + List result = new ArrayList<>(); + for (JsonElement param : getProperParameters(function)) { + result.add(convertChildAsType(param.getAsJsonObject(), "type")); + } + return result; + } + + private List convertParameterFields(JsonObject function) throws ParseError { + List result = new ArrayList<>(); + int index = -1; + for (JsonElement paramElm : getProperParameters(function)) { + ++index; + JsonObject param = paramElm.getAsJsonObject(); + int flags = getMemberModifierKeywords(param); + if (flags == DeclarationFlags.none) { + // If there are no flags, this is not a field parameter. + continue; + } + // We generate a synthetic field node, but do not copy any of the AST nodes from + // the parameter. The QL library overrides accessors to the name and type + // annotation to return those from the corresponding parameter. + SourceLocation loc = getSourceLocation(param); + if (param.has("initializer")) { + // Do not include the default parameter value in the source range for the field. + SourceLocation endLoc; + if (param.has("type")) { + endLoc = getSourceLocation(param.get("type").getAsJsonObject()); + } else { + endLoc = getSourceLocation(param.get("name").getAsJsonObject()); + } + loc.setEnd(endLoc.getEnd()); + loc.setSource(source.substring(loc.getStart().getOffset(), loc.getEnd().getOffset())); + } + FieldDefinition field = new FieldDefinition(loc, flags, null, null, null, index); + result.add(field); + } + return result; + } + + private Node convertParameter(JsonObject node, SourceLocation loc) throws ParseError { + // Note that type annotations are not extracted in this function, but in a + // separate pass in convertParameterTypes above. + Expression name = convertChild(node, "name", "Identifier"); + if (hasChild(node, "dotDotDotToken")) return new RestElement(loc, name); + if (hasChild(node, "initializer")) + return new AssignmentPattern(loc, "=", name, convertChild(node, "initializer")); + return name; + } + + private Node convertParenthesizedExpression(JsonObject node, SourceLocation loc) + throws ParseError { + return new ParenthesizedExpression(loc, convertChild(node, "expression")); + } + + private Node convertParenthesizedType(JsonObject node, SourceLocation loc) throws ParseError { + return new ParenthesizedTypeExpr(loc, convertChildAsType(node, "type")); + } + + private Node convertPostfixUnaryExpression(JsonObject node, SourceLocation loc) + throws ParseError { + String operator = getOperator(node); + return new UpdateExpression(loc, operator, convertChild(node, "operand"), false); + } + + private Node convertPrefixUnaryExpression(JsonObject node, SourceLocation loc) throws ParseError { + String operator = getOperator(node); + if ("++".equals(operator) || "--".equals(operator)) + return new UpdateExpression(loc, operator, convertChild(node, "operand"), true); + else return new UnaryExpression(loc, operator, convertChild(node, "operand"), true); + } + + private String getOperator(JsonObject node) throws ParseError { + int operatorId = node.get("operator").getAsInt(); + switch (syntaxKindMap.get(operatorId)) { + case "PlusPlusToken": + return "++"; + case "MinusMinusToken": + return "--"; + case "PlusToken": + return "+"; + case "MinusToken": + return "-"; + case "TildeToken": + return "~"; + case "ExclamationToken": + return "!"; + default: + throw new ParseError( + "Unsupported TypeScript operator " + operatorId, getSourceLocation(node).getStart()); + } + } + + private Node convertPropertyAccessExpression(JsonObject node, SourceLocation loc) + throws ParseError { + return new MemberExpression( + loc, convertChild(node, "expression"), convertChild(node, "name"), false, false, false); + } + + private Node convertPropertyAssignment(JsonObject node, SourceLocation loc) throws ParseError { + return new Property( + loc, + convertChild(node, "name"), + convertChild(node, "initializer"), + "init", + hasComputedName(node), + false); + } + + private Node convertPropertyDeclaration(JsonObject node, String kind, SourceLocation loc) + throws ParseError { + int flags = getMemberModifierKeywords(node); + if (hasComputedName(node)) { + flags |= DeclarationFlags.computed; + } + if (kind.equals("PropertySignature")) { + flags |= DeclarationFlags.abstract_; + } + if (node.get("questionToken") != null) { + flags |= DeclarationFlags.optional; + } + if (node.get("exclamationToken") != null) { + flags |= DeclarationFlags.definiteAssignmentAssertion; + } + FieldDefinition fieldDefinition = + new FieldDefinition( + loc, + flags, + convertChild(node, "name"), + convertChild(node, "initializer"), + convertChildAsType(node, "type")); + if (node.has("decorators")) { + fieldDefinition.addDecorators(convertChildren(node, "decorators")); + advanceUntilAfter(loc, fieldDefinition.getDecorators()); + } + return fieldDefinition; + } + + private Node convertRegularExpressionLiteral(SourceLocation loc) { + return new Literal(loc, TokenType.regexp, null); + } + + private Node convertRestType(JsonObject node, SourceLocation loc) throws ParseError { + return new RestTypeExpr(loc, convertChild(node, "type")); + } + + private Node convertQualifiedName(JsonObject node, SourceLocation loc) throws ParseError { + MemberExpression expr = + new MemberExpression( + loc, convertChild(node, "left"), convertChild(node, "right"), false, false, false); + attachSymbolInformation(expr, node); + return expr; + } + + private Node convertReturnStatement(JsonObject node, SourceLocation loc) throws ParseError { + return new ReturnStatement(loc, convertChild(node, "expression")); + } + + private Node convertSemicolonClassElement() { + return null; + } + + private Node convertSourceFile(JsonObject node, SourceLocation loc) throws ParseError { + List statements = convertNodes(node.get("statements").getAsJsonArray()); + Program program = new Program(loc, statements, "module"); + attachSymbolInformation(program, node); + return program; + } + + private Node convertShorthandPropertyAssignment(JsonObject node, SourceLocation loc) + throws ParseError { + return new Property( + loc, convertChild(node, "name"), convertChild(node, "name"), "init", false, false); + } + + private Node convertSpreadElement(JsonObject node, SourceLocation loc) throws ParseError { + return new SpreadElement(loc, convertChild(node, "expression")); + } + + private Node convertStringLiteral(JsonObject node, SourceLocation loc) { + return new Literal(loc, TokenType.string, node.get("text").getAsString()); + } + + private Node convertSuperKeyword(SourceLocation loc) { + return new Super(loc); + } + + private Node convertSwitchStatement(JsonObject node, SourceLocation loc) throws ParseError { + JsonObject caseBlock = node.get("caseBlock").getAsJsonObject(); + return new SwitchStatement( + loc, convertChild(node, "expression"), convertChildren(caseBlock, "clauses")); + } + + private Node convertTaggedTemplateExpression(JsonObject node, SourceLocation loc) + throws ParseError { + return new TaggedTemplateExpression( + loc, convertChild(node, "tag"), convertChild(node, "template")); + } + + private Node convertTemplateExpression(JsonObject node, SourceLocation loc) throws ParseError { + List quasis; + List expressions = new ArrayList<>(); + quasis = new ArrayList<>(); + quasis.add(convertChild(node, "head")); + for (JsonElement elt : node.get("templateSpans").getAsJsonArray()) { + JsonObject templateSpan = (JsonObject) elt; + expressions.add(convertChild(templateSpan, "expression")); + quasis.add(convertChild(templateSpan, "literal")); + } + return new TemplateLiteral(loc, expressions, quasis); + } + + private Node convertTemplateElement(JsonObject node, String kind, SourceLocation loc) { + boolean tail = "TemplateTail".equals(kind); + if (loc.getSource().startsWith("`") || loc.getSource().startsWith("}")) { + loc.setSource(loc.getSource().substring(1)); + Position start = loc.getStart(); + loc.setStart(new Position(start.getLine(), start.getColumn() + 1, start.getColumn() + 1)); + } + if (loc.getSource().endsWith("${")) { + loc.setSource(loc.getSource().substring(0, loc.getSource().length() - 2)); + Position end = loc.getEnd(); + loc.setEnd(new Position(end.getLine(), end.getColumn() - 2, end.getColumn() - 2)); + } + if (loc.getSource().endsWith("`")) { + loc.setSource(loc.getSource().substring(0, loc.getSource().length() - 1)); + Position end = loc.getEnd(); + loc.setEnd(new Position(end.getLine(), end.getColumn() - 1, end.getColumn() - 1)); + } + return new TemplateElement(loc, node.get("text").getAsString(), loc.getSource(), tail); + } + + private Node convertThisKeyword(SourceLocation loc) { + return new ThisExpression(loc); + } + + private Node convertThrowStatement(JsonObject node, SourceLocation loc) throws ParseError { + Expression expr = convertChild(node, "expression"); + if (expr == null) return convertEmptyStatement(loc); + return new ThrowStatement(loc, expr); + } + + private Node convertTrueKeyword(SourceLocation loc) { + return new Literal(loc, TokenType._true, true); + } + + private Node convertTryStatement(JsonObject node, SourceLocation loc) throws ParseError { + return new TryStatement( + loc, + convertChild(node, "tryBlock"), + convertChild(node, "catchClause"), + null, + convertChild(node, "finallyBlock")); + } + + private Node convertTupleType(JsonObject node, SourceLocation loc) throws ParseError { + return new TupleTypeExpr(loc, convertChildrenAsTypes(node, "elementTypes")); + } + + private Node convertTypeAliasDeclaration(JsonObject node, SourceLocation loc) throws ParseError { + TypeAliasDeclaration typeAlias = + new TypeAliasDeclaration( + loc, + convertChild(node, "name"), + convertChildrenNotNull(node, "typeParameters"), + convertChildAsType(node, "type")); + attachSymbolInformation(typeAlias, node); + return fixExports(loc, typeAlias); + } + + private Node convertTypeAssertionExpression(JsonObject node, SourceLocation loc) + throws ParseError { + return new TypeAssertion( + loc, convertChild(node, "expression"), convertChildAsType(node, "type"), false); + } + + private Node convertTypeLiteral(JsonObject obj, SourceLocation loc) throws ParseError { + return new InterfaceTypeExpr(loc, convertChildren(obj, "members")); + } + + private Node convertTypeOfExpression(JsonObject node, SourceLocation loc) throws ParseError { + return new UnaryExpression(loc, "typeof", convertChild(node, "expression"), true); + } + + private Node convertTypeOperator(JsonObject node, SourceLocation loc) throws ParseError { + String operator = syntaxKinds.get("" + node.get("operator").getAsInt()).getAsString(); + if (operator.equals("KeyOfKeyword")) { + return new KeyofTypeExpr(loc, convertChildAsType(node, "type")); + } + if (operator.equals("UniqueKeyword")) { + return new KeywordTypeExpr(loc, "unique symbol"); + } + throw new ParseError("Unsupported TypeScript syntax", loc.getStart()); + } + + private Node convertTypeParameter(JsonObject node, SourceLocation loc) throws ParseError { + return new TypeParameter( + loc, + convertChild(node, "name"), + convertChildAsType(node, "constraint"), + convertChildAsType(node, "default")); + } + + private Node convertTypePredicate(JsonObject node, SourceLocation loc) throws ParseError { + return new IsTypeExpr( + loc, convertChildAsType(node, "parameterName"), convertChildAsType(node, "type")); + } + + private Node convertTypeReference(JsonObject node, SourceLocation loc) throws ParseError { + ITypeExpression typeName = convertChild(node, "typeName"); + List typeArguments = convertChildrenAsTypes(node, "typeArguments"); + if (typeArguments.isEmpty()) return (Node) typeName; + return new GenericTypeExpr(loc, typeName, typeArguments); + } + + private Node convertTypeQuery(JsonObject node, SourceLocation loc) throws ParseError { + return new TypeofTypeExpr(loc, convertChildAsType(node, "exprName")); + } + + private Node convertVariableDeclaration(JsonObject node, SourceLocation loc) throws ParseError { + return new VariableDeclarator( + loc, + convertChild(node, "name"), + convertChild(node, "initializer"), + convertChildAsType(node, "type"), + DeclarationFlags.getDefiniteAssignmentAssertion(node.get("exclamationToken") != null)); + } + + private Node convertVariableDeclarationList(JsonObject node, SourceLocation loc) + throws ParseError { + return new VariableDeclaration( + loc, getDeclarationKind(node), convertVariableDeclarations(node), false); + } + + private List convertVariableDeclarations(JsonObject node) throws ParseError { + if (node.get("declarations").getAsJsonArray().size() == 0) + throw new ParseError("Unexpected token", getSourceLocation(node).getEnd()); + return convertChildren(node, "declarations"); + } + + private Node convertVariableStatement(JsonObject node, SourceLocation loc) throws ParseError { + JsonObject declarationList = node.get("declarationList").getAsJsonObject(); + String declarationKind = getDeclarationKind(declarationList); + List declarations = convertVariableDeclarations(declarationList); + boolean hasDeclareKeyword = hasModifier(node, "DeclareKeyword"); + VariableDeclaration vd = + new VariableDeclaration(loc, declarationKind, declarations, hasDeclareKeyword); + return fixExports(loc, vd); + } + + private Node convertVoidExpression(JsonObject node, SourceLocation loc) throws ParseError { + return new UnaryExpression(loc, "void", convertChild(node, "expression"), true); + } + + private Node convertWhileStatement(JsonObject node, SourceLocation loc) throws ParseError { + return new WhileStatement( + loc, convertChild(node, "expression"), convertChild(node, "statement")); + } + + private Node convertWithStatement(JsonObject node, SourceLocation loc) throws ParseError { + return new WithStatement( + loc, convertChild(node, "expression"), convertChild(node, "statement")); + } + + private Node convertYieldExpression(JsonObject node, SourceLocation loc) throws ParseError { + return new YieldExpression( + loc, convertChild(node, "expression"), hasChild(node, "asteriskToken")); + } + + /** + * Convert {@code e} to an lvalue expression, replacing {@link ArrayExpression} with {@link + * ArrayPattern}, {@link AssignmentExpression} with {@link AssignmentPattern}, {@link + * ObjectExpression} with {@link ObjectPattern} and {@link SpreadElement} with {@link + * RestElement}. + */ + private Expression convertLValue(Expression e) { + if (e == null) return null; + + SourceLocation loc = e.getLoc(); + if (e instanceof ArrayExpression) { + List elts = new ArrayList(); + for (Expression elt : ((ArrayExpression) e).getElements()) elts.add(convertLValue(elt)); + return new ArrayPattern(loc, elts); + } + if (e instanceof AssignmentExpression) { + AssignmentExpression a = (AssignmentExpression) e; + return new AssignmentPattern(loc, a.getOperator(), convertLValue(a.getLeft()), a.getRight()); + } + if (e instanceof ObjectExpression) { + List props = new ArrayList(); + for (Property prop : ((ObjectExpression) e).getProperties()) { + Expression key = prop.getKey(); + Expression rawValue = prop.getRawValue(); + String kind = prop.getKind().name(); + boolean isComputed = prop.isComputed(); + boolean isMethod = prop.isMethod(); + props.add( + new Property(prop.getLoc(), key, convertLValue(rawValue), kind, isComputed, isMethod)); + } + return new ObjectPattern(loc, props); + } + if (e instanceof ParenthesizedExpression) + return new ParenthesizedExpression( + loc, convertLValue(((ParenthesizedExpression) e).getExpression())); + if (e instanceof SpreadElement) + return new RestElement(e.getLoc(), convertLValue(((SpreadElement) e).getArgument())); + return e; + } + + /** Convert {@code e} to an {@link IJSXName}. */ + private IJSXName convertJSXName(Expression e) { + if (e instanceof Identifier) return new JSXIdentifier(e.getLoc(), ((Identifier) e).getName()); + if (e instanceof MemberExpression) { + MemberExpression me = (MemberExpression) e; + return new JSXMemberExpression( + e.getLoc(), + convertJSXName(me.getObject()), + (JSXIdentifier) convertJSXName(me.getProperty())); + } + if (e instanceof ThisExpression) return new JSXIdentifier(e.getLoc(), "this"); + return (IJSXName) e; + } + + /** + * Check whether {@code decl} has an {@code export} annotation, and if so wrap it inside an {@link + * ExportDeclaration}. + * + *

      If the declared statement has decorators, the {@code loc} should first be advanced past + * these using {@link #advanceUntilAfter}. + */ + private Node fixExports(SourceLocation loc, Statement decl) { + Matcher m = EXPORT_DECL_START.matcher(loc.getSource()); + if (m.find()) { + String skipped = m.group(0); + SourceLocation outerLoc = new SourceLocation(loc.getSource(), loc.getStart(), loc.getEnd()); + advance(loc, skipped); + // capture group 1 is `default`, if present + if (m.group(1) == null) + return new ExportNamedDeclaration(outerLoc, decl, new ArrayList<>(), null); + return new ExportDefaultDeclaration(outerLoc, decl); + } + return decl; + } + + /** Holds if the {@code name} property of the given AST node is a computed property name. */ + private boolean hasComputedName(JsonObject node) { + return hasComputedName(node, "name"); + } + + /** Holds if the given property of the given AST node is a computed property name. */ + private boolean hasComputedName(JsonObject node, String propName) { + return hasKind(node.get(propName), "ComputedPropertyName"); + } + + /** + * Update the start position and source text of {@code loc} by skipping over the string {@code + * skipped}. + */ + private void advance(SourceLocation loc, String skipped) { + loc.setStart(advance(loc.getStart(), skipped)); + loc.setSource(loc.getSource().substring(skipped.length())); + } + + /** + * Update the start position of @{code loc} by skipping over the given children and any following + * whitespace and comments, provided they are contained in the source location. + */ + private void advanceUntilAfter(SourceLocation loc, List nodes) { + if (nodes.isEmpty()) return; + INode last = nodes.get(nodes.size() - 1); + int offset = last.getLoc().getEnd().getOffset() - loc.getStart().getOffset(); + if (offset <= 0) return; + offset += matchWhitespace(last.getLoc().getEnd().getOffset()).length(); + if (offset >= loc.getSource().length()) return; + loc.setStart(advance(loc.getStart(), loc.getSource().substring(0, offset))); + loc.setSource(loc.getSource().substring(offset)); + } + + /** Get the longest sequence of whitespace or comment characters starting at the given offset. */ + private String matchWhitespace(int offset) { + Matcher m = WHITESPACE.matcher(source.substring(offset)); + m.find(); + return m.group(0); + } + + /** + * Create a position corresponding to {@code pos}, but updated by skipping over the string {@code + * skipped}. + */ + private Position advance(Position pos, String skipped) { + int innerStartOffset = pos.getOffset() + skipped.length(); + int innerStartLine = pos.getLine(), innerStartColumn = pos.getColumn(); + Matcher m = LINE_TERMINATOR.matcher(skipped); + int lastEnd = 0; + while (m.find()) { + ++innerStartLine; + innerStartColumn = 1; + lastEnd = m.end(); + } + innerStartColumn += skipped.length() - lastEnd; + if (lastEnd > 0) --innerStartColumn; + Position innerStart = new Position(innerStartLine, innerStartColumn, innerStartOffset); + return innerStart; + } + + /** Get the source location of the given AST node. */ + private SourceLocation getSourceLocation(JsonObject node) { + Position start = getPosition(node.get("$pos")); + Position end = getPosition(node.get("$end")); + int startOffset = start.getOffset(); + int endOffset = end.getOffset(); + if (startOffset > endOffset) startOffset = endOffset; + if (endOffset > source.length()) endOffset = source.length(); + return new SourceLocation(source.substring(startOffset, endOffset), start, end); + } + + /** + * Convert the given position object into a {@link Position}. For start positions, we need to skip + * over whitespace, which is included in the positions reported by the TypeScript compiler. + */ + private Position getPosition(JsonElement elm) { + int offset = elm.getAsInt(); + int line = getLineFromPos(offset); + int column = getColumnFromLinePos(line, offset); + return new Position(line + 1, column, offset); + } + + private Iterable getModifiers(JsonObject node) { + JsonElement mods = node.get("modifiers"); + if (!(mods instanceof JsonArray)) return Collections.emptyList(); + return (JsonArray) mods; + } + + /** + * Returns a specific modifier from the given node (or null if absent), as defined by its + * modifiers property and the kind property of the modifier AST node. + */ + private JsonObject getModifier(JsonObject node, String modKind) { + for (JsonElement mod : getModifiers(node)) + if (mod instanceof JsonObject) + if (hasKind((JsonObject) mod, modKind)) return (JsonObject) mod; + return null; + } + + /** + * Check whether a node has a particular modifier, as defined by its modifiers property + * and the kind property of the modifier AST node. + */ + private boolean hasModifier(JsonObject node, String modKind) { + return getModifier(node, modKind) != null; + } + + private int getDeclarationModifierFromKeyword(String kind) { + switch (kind) { + case "AbstractKeyword": + return DeclarationFlags.abstract_; + case "StaticKeyword": + return DeclarationFlags.static_; + case "ReadonlyKeyword": + return DeclarationFlags.readonly; + case "PublicKeyword": + return DeclarationFlags.public_; + case "PrivateKeyword": + return DeclarationFlags.private_; + case "ProtectedKeyword": + return DeclarationFlags.protected_; + default: + return DeclarationFlags.none; + } + } + + /** + * Returns the set of member flags corresponding to the modifier keywords present on the given + * node. + */ + private int getMemberModifierKeywords(JsonObject node) { + int flags = DeclarationFlags.none; + for (JsonElement mod : getModifiers(node)) { + if (mod instanceof JsonObject) { + JsonObject modObject = (JsonObject) mod; + flags |= getDeclarationModifierFromKeyword(getKind(modObject)); + } + } + return flags; + } + + /** + * Check whether a node has a particular flag, as defined by its flags property and the + * ts.NodeFlags in enum. + */ + private boolean hasFlag(JsonObject node, String flagName) { + JsonElement flagDescriptor = this.nodeFlags.get(flagName); + if (flagDescriptor == null) { + throw new RuntimeException( + "Incompatible version of TypeScript installed. Missing node flag " + flagName); + } + int flagId = flagDescriptor.getAsInt(); + JsonElement flags = node.get("flags"); + if (flags instanceof JsonPrimitive) { + return (flags.getAsInt() & flagId) != 0; + } + return false; + } + + /** Gets the numeric value of the syntax kind enum with the given name. */ + private int getSyntaxKind(String syntaxKind) { + JsonElement descriptor = this.syntaxKinds.get(syntaxKind); + if (descriptor == null) { + throw new RuntimeException( + "Incompatible version of TypeScript installed. Missing syntax kind " + syntaxKind); + } + return descriptor.getAsInt(); + } + + /** Check whether a node has a child with a given name. */ + private boolean hasChild(JsonObject node, String prop) { + if (!node.has(prop)) return false; + return !(node.get(prop) instanceof JsonNull); + } + + /** + * Returns an iterator over the elements of the given child array, or an empty iterator if the + * given child is not an array. + */ + private Iterable getChildIterable(JsonObject node, String child) { + JsonElement elt = node.get(child); + if (!(elt instanceof JsonArray)) return Collections.emptyList(); + return (JsonArray) elt; + } + + /** Gets the kind of the given node. */ + private String getKind(JsonElement node) { + if (node instanceof JsonObject) { + JsonElement kind = ((JsonObject) node).get("kind"); + if (kind instanceof JsonPrimitive && ((JsonPrimitive) kind).isNumber()) + return syntaxKindMap.get(kind.getAsInt()); + } + return null; + } + + /** Holds if the given node has the given kind. */ + private boolean hasKind(JsonElement node, String kind) { + return kind.equals(getKind(node)); + } + + /** + * Gets the declaration kind of the given node, which is one of {@code "var"}, {@code "let"} or + * {@code "const"}. + */ + private String getDeclarationKind(JsonObject declarationList) { + return declarationList.get("$declarationKind").getAsString(); + } } diff --git a/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java b/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java index bc956b34def6..6aca6e3a1f5e 100644 --- a/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java +++ b/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java @@ -1,19 +1,6 @@ package com.semmle.js.parser; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.ByteArrayOutputStream; -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.lang.ProcessBuilder.Redirect; -import java.util.ArrayList; -import java.util.List; - +import ch.qos.logback.classic.Level; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -34,399 +21,412 @@ import com.semmle.util.process.AbstractProcessBuilder; import com.semmle.util.process.Builder; import com.semmle.util.process.Env; - -import ch.qos.logback.classic.Level; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.lang.ProcessBuilder.Redirect; +import java.util.ArrayList; +import java.util.List; /** * The Java half of our wrapper for invoking the TypeScript parser. * - * The Node.js half of the wrapper is expected to live at - * {@code $SEMMLE_DIST/tools/typescript-parser-wrapper/main.js}; non-standard - * locations can be configured using the property {@link #PARSER_WRAPPER_PATH_ENV_VAR}. + *

      The Node.js half of the wrapper is expected to live at {@code + * $SEMMLE_DIST/tools/typescript-parser-wrapper/main.js}; non-standard locations can be configured + * using the property {@link #PARSER_WRAPPER_PATH_ENV_VAR}. * - * The script is started upon parsing the first TypeScript file and then is - * kept running in the background, passing it requests for parsing files and - * getting JSON-encoded ASTs as responses. + *

      The script is started upon parsing the first TypeScript file and then is kept running in the + * background, passing it requests for parsing files and getting JSON-encoded ASTs as responses. */ public class TypeScriptParser { - /** - * An environment variable that can be set to indicate the location of the - * TypeScript parser wrapper when running without SEMMLE_DIST. - */ - public static final String PARSER_WRAPPER_PATH_ENV_VAR = "SEMMLE_TYPESCRIPT_PARSER_WRAPPER"; - - /** - * An environment variable that can be set to specify a timeout to use when - * verifying the TypeScript installation, in milliseconds. Default is 10000. - */ - public static final String TYPESCRIPT_TIMEOUT_VAR = "SEMMLE_TYPESCRIPT_TIMEOUT"; - - /** - * An environment variable (without the SEMMLE_ or LGTM_ - * prefix), that can be set to indicate the maximum heap space usable by the - * Node.js process, in addition to its "reserve memory". - *

      - * Defaults to 1.0 GB (for a total heap space of 1.4 GB by default). - */ - public static final String TYPESCRIPT_RAM_SUFFIX = "TYPESCRIPT_RAM"; - - /** - * An environment variable (without the SEMMLE_ or LGTM_ - * prefix), that can be set to indicate the amount of heap space the Node.js - * process should reserve for extracting individual files. - *

      - * When less than this amount of memory is available, the TypeScript compiler - * instance is restarted to free space. - *

      - * Defaults to 400 MB (for a total heap space of 1.4 GB by default). - */ - public static final String TYPESCRIPT_RAM_RESERVE_SUFFIX = "TYPESCRIPT_RAM_RESERVE"; - - /** The Node.js parser wrapper process, if it has been started already. */ - private Process parserWrapperProcess; - private String parserWrapperCommand; - - /** Streams for communicating with the Node.js parser wrapper process. */ - private BufferedWriter toParserWrapper; - private BufferedReader fromParserWrapper; - - private String nodeJsVersionString; - - /** - * If non-zero, we use this instead of relying on the corresponding environment - * variable. - */ - private int typescriptRam = 0; - - /** - * Sets the amount of RAM to allocate to the TypeScript compiler.s - */ - public void setTypescriptRam(int megabytes) { - this.typescriptRam = megabytes; - } - - /** - * Verifies that Node.js and TypeScript are installed and throws an exception - * otherwise. - * - * @param verbose - * if true, log the version strings and NODE_PATH. - */ - public void verifyInstallation(boolean verbose) { - verifyNodeInstallation(); - if (verbose) { - System.out.println("Found Node.js version: " + nodeJsVersionString); - } - } - - /** - * Checks that Node.js is installed and can be run and returns its version - * string. - */ - public String verifyNodeInstallation() { - if (nodeJsVersionString != null) - return nodeJsVersionString; - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - ByteArrayOutputStream err = new ByteArrayOutputStream(); - Builder b = new Builder(out, err, getParserWrapper().getParentFile(), "node", "--version"); - b.expectFailure(); // We want to do our own logging in case of an error. - - int timeout = Env.systemEnv().getInt(TYPESCRIPT_TIMEOUT_VAR, 10000); - try { - int r = b.execute(timeout); - String stdout = new String(out.toByteArray()); - String stderr = new String(err.toByteArray()); - if (r != 0 || stdout.length() == 0) { - throw new CatastrophicError("Could not start Node.js. It is required for TypeScript extraction.\n" + stderr); - } - return nodeJsVersionString = stdout; - } catch (InterruptedError e) { - Exceptions.ignore(e, "Exception details are not important."); - throw new CatastrophicError("Could not start Node.js (timed out after " + (timeout / 1000) + "s)."); - } catch (ResourceError e) { - // In case 'node' is not found, the process builder converts the IOException - // into a ResourceError. - Exceptions.ignore(e, "We rewrite this into a UserError"); - throw new UserError("Could not start Node.js. It is required for TypeScript extraction." - + "\nPlease install Node.js and ensure 'node' is on the PATH."); - } - } - - private static int getMegabyteCountFromPrefixedEnv(String suffix, int defaultValue) { - String envVar = "SEMMLE_" + suffix; - String value = Env.systemEnv().get(envVar); - if (value == null || value.length() == 0) { - envVar = "LGTM_" + suffix; - value = Env.systemEnv().get(envVar); - } - if (value == null || value.length() == 0) { - return defaultValue; - } - Integer amount = UnitParser.parseOpt(value, UnitParser.MEGABYTES); - if (amount == null) { - throw new UserError("Invalid value for " + envVar + ": '" + value + "'"); - } - return amount; - } - - /** - * Start the Node.js parser wrapper process. - */ - private void setupParserWrapper() { - verifyNodeInstallation(); - - int mainMemoryMb = typescriptRam != 0 ? typescriptRam : getMegabyteCountFromPrefixedEnv(TYPESCRIPT_RAM_SUFFIX, 1000); - int reserveMemoryMb = getMegabyteCountFromPrefixedEnv(TYPESCRIPT_RAM_RESERVE_SUFFIX, 400); - - File parserWrapper = getParserWrapper(); - List cmd = new ArrayList<>(); - cmd.add("node"); - cmd.add("--max_old_space_size=" + (mainMemoryMb + reserveMemoryMb)); - cmd.add(parserWrapper.getAbsolutePath()); - ProcessBuilder pb = new ProcessBuilder(cmd); - parserWrapperCommand = StringUtil.glue(" ", cmd); - pb.environment().put("SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD", "" + mainMemoryMb); - - try { - pb.redirectError(Redirect.INHERIT); // Forward stderr to our own stderr. - parserWrapperProcess = pb.start(); - OutputStream os = parserWrapperProcess.getOutputStream(); - OutputStreamWriter osw = new OutputStreamWriter(os, "UTF-8"); - toParserWrapper = new BufferedWriter(osw); - InputStream is = parserWrapperProcess.getInputStream(); - InputStreamReader isr = new InputStreamReader(is, "UTF-8"); - fromParserWrapper = new BufferedReader(isr); - } catch (IOException e) { - throw new CatastrophicError("Could not start TypeScript parser wrapper " - + "(command: ." + parserWrapperCommand + ")", e); - } - } - - /** - * Get the location of the Node.js parser wrapper script. - */ - private File getParserWrapper() { - File parserWrapper; - LogbackUtils.getLogger(AbstractProcessBuilder.class).setLevel(Level.INFO); - String explicitPath = Env.systemEnv().get(PARSER_WRAPPER_PATH_ENV_VAR); - String semmleDistVar = Env.systemEnv().get(Env.Var.SEMMLE_DIST.name()); - if (semmleDistVar != null && !semmleDistVar.isEmpty()) { - parserWrapper = new File(semmleDistVar, "tools/typescript-parser-wrapper/main.js"); - } else if (explicitPath != null) { - parserWrapper = new File(explicitPath); - } else { - throw new CatastrophicError("Could not find TypeScript parser: " + Env.Var.SEMMLE_DIST.name() + " is not set."); - } - if (!parserWrapper.isFile()) - throw new ResourceError("Could not find TypeScript parser: " + - parserWrapper + " does not exist."); - return parserWrapper; - } - - /** - * Send a {@code request} to the Node.js parser wrapper process, and return - * the response it replies with. - */ - private JsonObject talkToParserWrapper(JsonObject request) { - if (parserWrapperProcess == null) - setupParserWrapper(); - - if (!parserWrapperProcess.isAlive()) { - int exitCode = 0; - try { - exitCode = parserWrapperProcess.waitFor(); - } catch (InterruptedException e) { - Exceptions.ignore(e, "This is for diagnostic purposes only."); - } - String err = new WholeIO().strictReadString(parserWrapperProcess.getErrorStream()); - throw new CatastrophicError("TypeScript parser wrapper terminated with exit code " + - exitCode + "; stderr: " + err); - } - - String response = null; - try { - toParserWrapper.write(request.toString()); - toParserWrapper.newLine(); - toParserWrapper.flush(); - response = fromParserWrapper.readLine(); - if (response == null) - throw new CatastrophicError( - "Could not communicate with TypeScript parser wrapper " - + "(command: " + parserWrapperCommand + ")."); - return new JsonParser().parse(response).getAsJsonObject(); - } catch (IOException e) { - throw new CatastrophicError( - "Could not communicate with TypeScript parser wrapper " - + "(command: ." + parserWrapperCommand + ").", e); - } catch (JsonParseException | IllegalStateException e) { - throw new CatastrophicError( - "TypeScript parser wrapper sent unexpected response: " + - response + " (command: " + parserWrapperCommand + ").", e); - } - } - - /** - * Returns the AST for a given source file. - *

      - * Type information will be available if the file is part of a currently open - * project, although this is not yet implemented. - *

      - * If the file is not part of a project, only syntactic information will be - * extracted. - */ - public Result parse(File sourceFile, String source) { - JsonObject request = new JsonObject(); - request.add("command", new JsonPrimitive("parse")); - request.add("filename", new JsonPrimitive(sourceFile.getAbsolutePath())); - JsonObject response = talkToParserWrapper(request); - try { - checkResponseType(response, "ast"); - JsonObject nodeFlags = response.get("nodeFlags").getAsJsonObject(); - JsonObject syntaxKinds = response.get("syntaxKinds").getAsJsonObject(); - JsonObject ast = response.get("ast").getAsJsonObject(); - return new TypeScriptASTConverter(nodeFlags, syntaxKinds).convertAST(ast, source); - } catch (IllegalStateException e) { - throw new CatastrophicError("TypeScript parser wrapper sent unexpected response: " + - response, e); - } - } - - /** - * Informs the parser process that the following files are going to be - * requested, in that order. - *

      - * The parser process uses this list to start work on the next file before it is - * requested. - */ - public void prepareFiles(List files) { - JsonObject request = new JsonObject(); - request.add("command", new JsonPrimitive("prepare-files")); - JsonArray filenames = new JsonArray(); - for (File file : files) { - filenames.add(new JsonPrimitive(file.getAbsolutePath())); - } - request.add("filenames", filenames); - JsonObject response = talkToParserWrapper(request); - checkResponseType(response, "ok"); - } - - /** - * Opens a new project based on a tsconfig.json file. The compiler will analyze - * all files in the project. - *

      - * Call {@link #parse} to access individual files in the project. - *

      - * Only one project should be opened at once. - */ - public ParsedProject openProject(File tsConfigFile) { - JsonObject request = new JsonObject(); - request.add("command", new JsonPrimitive("open-project")); - request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath())); - JsonObject response = talkToParserWrapper(request); - try { - checkResponseType(response, "project-opened"); - ParsedProject project = new ParsedProject(tsConfigFile); - JsonArray filesJson = response.get("files").getAsJsonArray(); - for (JsonElement elm : filesJson) { - project.addSourceFile(new File(elm.getAsString())); - } - return project; - } catch (IllegalStateException e) { - throw new CatastrophicError("TypeScript parser wrapper sent unexpected response: " + response, e); - } - } - - /** - * Closes a project previously opened. - *

      - * This main purpose is to free heap space in the Node.js process. - */ - public void closeProject(File tsConfigFile) { - JsonObject request = new JsonObject(); - request.add("command", new JsonPrimitive("close-project")); - request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath())); - JsonObject response = talkToParserWrapper(request); - try { - checkResponseType(response, "project-closed"); - } catch (IllegalStateException e) { - throw new CatastrophicError("TypeScript parser wrapper sent unexpected response: " + response, e); - } - } - - public TypeTable getTypeTable() { - JsonObject request = new JsonObject(); - request.add("command", new JsonPrimitive("get-type-table")); - JsonObject response = talkToParserWrapper(request); - try { - checkResponseType(response, "type-table"); - return new TypeTable(response.get("typeTable").getAsJsonObject()); - } catch (IllegalStateException e) { - throw new CatastrophicError("TypeScript parser wrapper sent unexpected response: " + response, e); - } - } - - /** - * Closes any open project, and in general, brings the TypeScript wrapper to a - * fresh state as if it had just been restarted. - *

      - * This is to ensure tests are isolated but without the cost of restarting the - * Node.js process. - */ - public void reset() { - try { - resetInternal(); - } catch (CatastrophicError e) { - Exceptions.ignore(e, "Restarting process instead"); - killProcess(); - } - } - - private void resetInternal() { - if (parserWrapperProcess == null) { - return; // Ignore reset requests if the process is not running. - } - JsonObject request = new JsonObject(); - request.add("command", new JsonPrimitive("reset")); - JsonObject response = talkToParserWrapper(request); - try { - checkResponseType(response, "reset-done"); - } catch (IllegalStateException e) { - throw new CatastrophicError("TypeScript parser wrapper sent unexpected response: " + response, e); - } - } - - private void checkResponseType(JsonObject response, String type) { - JsonElement typeElm = response.get("type"); - // Report unexpected response types as an internal error. - if (typeElm == null || !typeElm.getAsString().equals(type)) { - throw new CatastrophicError("TypeScript parser sent unexpected response: " + response + ". Expected " + type); - } - } - - private void tryClose(Closeable stream) { - if (stream == null) - return; - try { - stream.close(); - } catch (IOException e) { - Exceptions.ignore(e, "Closing stream"); - } - } - - /** - * Forcibly closes the Node.js process. - * - * A new process will be started the next time a request is made. - */ - public void killProcess() { - if (parserWrapperProcess != null) { - parserWrapperProcess.destroy(); - parserWrapperProcess = null; - } - tryClose(toParserWrapper); - tryClose(fromParserWrapper); - toParserWrapper = null; - fromParserWrapper = null; - } + /** + * An environment variable that can be set to indicate the location of the TypeScript parser + * wrapper when running without SEMMLE_DIST. + */ + public static final String PARSER_WRAPPER_PATH_ENV_VAR = "SEMMLE_TYPESCRIPT_PARSER_WRAPPER"; + + /** + * An environment variable that can be set to specify a timeout to use when verifying the + * TypeScript installation, in milliseconds. Default is 10000. + */ + public static final String TYPESCRIPT_TIMEOUT_VAR = "SEMMLE_TYPESCRIPT_TIMEOUT"; + + /** + * An environment variable (without the SEMMLE_ or LGTM_ prefix), that can be + * set to indicate the maximum heap space usable by the Node.js process, in addition to its + * "reserve memory". + * + *

      Defaults to 1.0 GB (for a total heap space of 1.4 GB by default). + */ + public static final String TYPESCRIPT_RAM_SUFFIX = "TYPESCRIPT_RAM"; + + /** + * An environment variable (without the SEMMLE_ or LGTM_ prefix), that can be + * set to indicate the amount of heap space the Node.js process should reserve for extracting + * individual files. + * + *

      When less than this amount of memory is available, the TypeScript compiler instance is + * restarted to free space. + * + *

      Defaults to 400 MB (for a total heap space of 1.4 GB by default). + */ + public static final String TYPESCRIPT_RAM_RESERVE_SUFFIX = "TYPESCRIPT_RAM_RESERVE"; + + /** The Node.js parser wrapper process, if it has been started already. */ + private Process parserWrapperProcess; + + private String parserWrapperCommand; + + /** Streams for communicating with the Node.js parser wrapper process. */ + private BufferedWriter toParserWrapper; + + private BufferedReader fromParserWrapper; + + private String nodeJsVersionString; + + /** If non-zero, we use this instead of relying on the corresponding environment variable. */ + private int typescriptRam = 0; + + /** Sets the amount of RAM to allocate to the TypeScript compiler.s */ + public void setTypescriptRam(int megabytes) { + this.typescriptRam = megabytes; + } + + /** + * Verifies that Node.js and TypeScript are installed and throws an exception otherwise. + * + * @param verbose if true, log the version strings and NODE_PATH. + */ + public void verifyInstallation(boolean verbose) { + verifyNodeInstallation(); + if (verbose) { + System.out.println("Found Node.js version: " + nodeJsVersionString); + } + } + + /** Checks that Node.js is installed and can be run and returns its version string. */ + public String verifyNodeInstallation() { + if (nodeJsVersionString != null) return nodeJsVersionString; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayOutputStream err = new ByteArrayOutputStream(); + Builder b = new Builder(out, err, getParserWrapper().getParentFile(), "node", "--version"); + b.expectFailure(); // We want to do our own logging in case of an error. + + int timeout = Env.systemEnv().getInt(TYPESCRIPT_TIMEOUT_VAR, 10000); + try { + int r = b.execute(timeout); + String stdout = new String(out.toByteArray()); + String stderr = new String(err.toByteArray()); + if (r != 0 || stdout.length() == 0) { + throw new CatastrophicError( + "Could not start Node.js. It is required for TypeScript extraction.\n" + stderr); + } + return nodeJsVersionString = stdout; + } catch (InterruptedError e) { + Exceptions.ignore(e, "Exception details are not important."); + throw new CatastrophicError( + "Could not start Node.js (timed out after " + (timeout / 1000) + "s)."); + } catch (ResourceError e) { + // In case 'node' is not found, the process builder converts the IOException + // into a ResourceError. + Exceptions.ignore(e, "We rewrite this into a UserError"); + throw new UserError( + "Could not start Node.js. It is required for TypeScript extraction." + + "\nPlease install Node.js and ensure 'node' is on the PATH."); + } + } + + private static int getMegabyteCountFromPrefixedEnv(String suffix, int defaultValue) { + String envVar = "SEMMLE_" + suffix; + String value = Env.systemEnv().get(envVar); + if (value == null || value.length() == 0) { + envVar = "LGTM_" + suffix; + value = Env.systemEnv().get(envVar); + } + if (value == null || value.length() == 0) { + return defaultValue; + } + Integer amount = UnitParser.parseOpt(value, UnitParser.MEGABYTES); + if (amount == null) { + throw new UserError("Invalid value for " + envVar + ": '" + value + "'"); + } + return amount; + } + + /** Start the Node.js parser wrapper process. */ + private void setupParserWrapper() { + verifyNodeInstallation(); + + int mainMemoryMb = + typescriptRam != 0 + ? typescriptRam + : getMegabyteCountFromPrefixedEnv(TYPESCRIPT_RAM_SUFFIX, 1000); + int reserveMemoryMb = getMegabyteCountFromPrefixedEnv(TYPESCRIPT_RAM_RESERVE_SUFFIX, 400); + + File parserWrapper = getParserWrapper(); + List cmd = new ArrayList<>(); + cmd.add("node"); + cmd.add("--max_old_space_size=" + (mainMemoryMb + reserveMemoryMb)); + cmd.add(parserWrapper.getAbsolutePath()); + ProcessBuilder pb = new ProcessBuilder(cmd); + parserWrapperCommand = StringUtil.glue(" ", cmd); + pb.environment().put("SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD", "" + mainMemoryMb); + + try { + pb.redirectError(Redirect.INHERIT); // Forward stderr to our own stderr. + parserWrapperProcess = pb.start(); + OutputStream os = parserWrapperProcess.getOutputStream(); + OutputStreamWriter osw = new OutputStreamWriter(os, "UTF-8"); + toParserWrapper = new BufferedWriter(osw); + InputStream is = parserWrapperProcess.getInputStream(); + InputStreamReader isr = new InputStreamReader(is, "UTF-8"); + fromParserWrapper = new BufferedReader(isr); + } catch (IOException e) { + throw new CatastrophicError( + "Could not start TypeScript parser wrapper " + "(command: ." + parserWrapperCommand + ")", + e); + } + } + + /** Get the location of the Node.js parser wrapper script. */ + private File getParserWrapper() { + File parserWrapper; + LogbackUtils.getLogger(AbstractProcessBuilder.class).setLevel(Level.INFO); + String explicitPath = Env.systemEnv().get(PARSER_WRAPPER_PATH_ENV_VAR); + String semmleDistVar = Env.systemEnv().get(Env.Var.SEMMLE_DIST.name()); + if (semmleDistVar != null && !semmleDistVar.isEmpty()) { + parserWrapper = new File(semmleDistVar, "tools/typescript-parser-wrapper/main.js"); + } else if (explicitPath != null) { + parserWrapper = new File(explicitPath); + } else { + throw new CatastrophicError( + "Could not find TypeScript parser: " + Env.Var.SEMMLE_DIST.name() + " is not set."); + } + if (!parserWrapper.isFile()) + throw new ResourceError( + "Could not find TypeScript parser: " + parserWrapper + " does not exist."); + return parserWrapper; + } + + /** + * Send a {@code request} to the Node.js parser wrapper process, and return the response it + * replies with. + */ + private JsonObject talkToParserWrapper(JsonObject request) { + if (parserWrapperProcess == null) setupParserWrapper(); + + if (!parserWrapperProcess.isAlive()) { + int exitCode = 0; + try { + exitCode = parserWrapperProcess.waitFor(); + } catch (InterruptedException e) { + Exceptions.ignore(e, "This is for diagnostic purposes only."); + } + String err = new WholeIO().strictReadString(parserWrapperProcess.getErrorStream()); + throw new CatastrophicError( + "TypeScript parser wrapper terminated with exit code " + exitCode + "; stderr: " + err); + } + + String response = null; + try { + toParserWrapper.write(request.toString()); + toParserWrapper.newLine(); + toParserWrapper.flush(); + response = fromParserWrapper.readLine(); + if (response == null) + throw new CatastrophicError( + "Could not communicate with TypeScript parser wrapper " + + "(command: " + + parserWrapperCommand + + ")."); + return new JsonParser().parse(response).getAsJsonObject(); + } catch (IOException e) { + throw new CatastrophicError( + "Could not communicate with TypeScript parser wrapper " + + "(command: ." + + parserWrapperCommand + + ").", + e); + } catch (JsonParseException | IllegalStateException e) { + throw new CatastrophicError( + "TypeScript parser wrapper sent unexpected response: " + + response + + " (command: " + + parserWrapperCommand + + ").", + e); + } + } + + /** + * Returns the AST for a given source file. + * + *

      Type information will be available if the file is part of a currently open project, although + * this is not yet implemented. + * + *

      If the file is not part of a project, only syntactic information will be extracted. + */ + public Result parse(File sourceFile, String source) { + JsonObject request = new JsonObject(); + request.add("command", new JsonPrimitive("parse")); + request.add("filename", new JsonPrimitive(sourceFile.getAbsolutePath())); + JsonObject response = talkToParserWrapper(request); + try { + checkResponseType(response, "ast"); + JsonObject nodeFlags = response.get("nodeFlags").getAsJsonObject(); + JsonObject syntaxKinds = response.get("syntaxKinds").getAsJsonObject(); + JsonObject ast = response.get("ast").getAsJsonObject(); + return new TypeScriptASTConverter(nodeFlags, syntaxKinds).convertAST(ast, source); + } catch (IllegalStateException e) { + throw new CatastrophicError( + "TypeScript parser wrapper sent unexpected response: " + response, e); + } + } + + /** + * Informs the parser process that the following files are going to be requested, in that order. + * + *

      The parser process uses this list to start work on the next file before it is requested. + */ + public void prepareFiles(List files) { + JsonObject request = new JsonObject(); + request.add("command", new JsonPrimitive("prepare-files")); + JsonArray filenames = new JsonArray(); + for (File file : files) { + filenames.add(new JsonPrimitive(file.getAbsolutePath())); + } + request.add("filenames", filenames); + JsonObject response = talkToParserWrapper(request); + checkResponseType(response, "ok"); + } + + /** + * Opens a new project based on a tsconfig.json file. The compiler will analyze all files in the + * project. + * + *

      Call {@link #parse} to access individual files in the project. + * + *

      Only one project should be opened at once. + */ + public ParsedProject openProject(File tsConfigFile) { + JsonObject request = new JsonObject(); + request.add("command", new JsonPrimitive("open-project")); + request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath())); + JsonObject response = talkToParserWrapper(request); + try { + checkResponseType(response, "project-opened"); + ParsedProject project = new ParsedProject(tsConfigFile); + JsonArray filesJson = response.get("files").getAsJsonArray(); + for (JsonElement elm : filesJson) { + project.addSourceFile(new File(elm.getAsString())); + } + return project; + } catch (IllegalStateException e) { + throw new CatastrophicError( + "TypeScript parser wrapper sent unexpected response: " + response, e); + } + } + + /** + * Closes a project previously opened. + * + *

      This main purpose is to free heap space in the Node.js process. + */ + public void closeProject(File tsConfigFile) { + JsonObject request = new JsonObject(); + request.add("command", new JsonPrimitive("close-project")); + request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath())); + JsonObject response = talkToParserWrapper(request); + try { + checkResponseType(response, "project-closed"); + } catch (IllegalStateException e) { + throw new CatastrophicError( + "TypeScript parser wrapper sent unexpected response: " + response, e); + } + } + + public TypeTable getTypeTable() { + JsonObject request = new JsonObject(); + request.add("command", new JsonPrimitive("get-type-table")); + JsonObject response = talkToParserWrapper(request); + try { + checkResponseType(response, "type-table"); + return new TypeTable(response.get("typeTable").getAsJsonObject()); + } catch (IllegalStateException e) { + throw new CatastrophicError( + "TypeScript parser wrapper sent unexpected response: " + response, e); + } + } + + /** + * Closes any open project, and in general, brings the TypeScript wrapper to a fresh state as if + * it had just been restarted. + * + *

      This is to ensure tests are isolated but without the cost of restarting the Node.js process. + */ + public void reset() { + try { + resetInternal(); + } catch (CatastrophicError e) { + Exceptions.ignore(e, "Restarting process instead"); + killProcess(); + } + } + + private void resetInternal() { + if (parserWrapperProcess == null) { + return; // Ignore reset requests if the process is not running. + } + JsonObject request = new JsonObject(); + request.add("command", new JsonPrimitive("reset")); + JsonObject response = talkToParserWrapper(request); + try { + checkResponseType(response, "reset-done"); + } catch (IllegalStateException e) { + throw new CatastrophicError( + "TypeScript parser wrapper sent unexpected response: " + response, e); + } + } + + private void checkResponseType(JsonObject response, String type) { + JsonElement typeElm = response.get("type"); + // Report unexpected response types as an internal error. + if (typeElm == null || !typeElm.getAsString().equals(type)) { + throw new CatastrophicError( + "TypeScript parser sent unexpected response: " + response + ". Expected " + type); + } + } + + private void tryClose(Closeable stream) { + if (stream == null) return; + try { + stream.close(); + } catch (IOException e) { + Exceptions.ignore(e, "Closing stream"); + } + } + + /** + * Forcibly closes the Node.js process. + * + *

      A new process will be started the next time a request is made. + */ + public void killProcess() { + if (parserWrapperProcess != null) { + parserWrapperProcess.destroy(); + parserWrapperProcess = null; + } + tryClose(toParserWrapper); + tryClose(fromParserWrapper); + toParserWrapper = null; + fromParserWrapper = null; + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/ArrayTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/ArrayTypeExpr.java index 56817707c146..badcf4444395 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/ArrayTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/ArrayTypeExpr.java @@ -4,23 +4,22 @@ import com.semmle.js.ast.Visitor; /** - * An array type, such as number[], or in general T[] where - * T is a type. + * An array type, such as number[], or in general T[] where T is a type. */ public class ArrayTypeExpr extends TypeExpression { - private final ITypeExpression elementType; + private final ITypeExpression elementType; - public ArrayTypeExpr(SourceLocation loc, ITypeExpression elementType) { - super("ArrayTypeExpr", loc); - this.elementType = elementType; - } + public ArrayTypeExpr(SourceLocation loc, ITypeExpression elementType) { + super("ArrayTypeExpr", loc); + this.elementType = elementType; + } - public ITypeExpression getElementType() { - return elementType; - } + public ITypeExpression getElementType() { + return elementType; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/ConditionalTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/ConditionalTypeExpr.java index 44643139e52c..61bc2c647f2e 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/ConditionalTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/ConditionalTypeExpr.java @@ -3,42 +3,44 @@ import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; -/** - * A conditional type annotation, such as T extends any[] ? A : B. - */ +/** A conditional type annotation, such as T extends any[] ? A : B. */ public class ConditionalTypeExpr extends TypeExpression { - private ITypeExpression checkType; - private ITypeExpression extendsType; - private ITypeExpression trueType; - private ITypeExpression falseType; - - public ConditionalTypeExpr(SourceLocation loc, ITypeExpression checkType, ITypeExpression extendsType, - ITypeExpression trueType, ITypeExpression falseType) { - super("ConditionalTypeExpr", loc); - this.checkType = checkType; - this.extendsType = extendsType; - this.trueType = trueType; - this.falseType = falseType; - } - - public ITypeExpression getCheckType() { - return checkType; - } - - public ITypeExpression getExtendsType() { - return extendsType; - } - - public ITypeExpression getTrueType() { - return trueType; - } - - public ITypeExpression getFalseType() { - return falseType; - } - - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + private ITypeExpression checkType; + private ITypeExpression extendsType; + private ITypeExpression trueType; + private ITypeExpression falseType; + + public ConditionalTypeExpr( + SourceLocation loc, + ITypeExpression checkType, + ITypeExpression extendsType, + ITypeExpression trueType, + ITypeExpression falseType) { + super("ConditionalTypeExpr", loc); + this.checkType = checkType; + this.extendsType = extendsType; + this.trueType = trueType; + this.falseType = falseType; + } + + public ITypeExpression getCheckType() { + return checkType; + } + + public ITypeExpression getExtendsType() { + return extendsType; + } + + public ITypeExpression getTrueType() { + return trueType; + } + + public ITypeExpression getFalseType() { + return falseType; + } + + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/DecoratorList.java b/javascript/extractor/src/com/semmle/ts/ast/DecoratorList.java index 5c50102085ad..5a74641fe4aa 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/DecoratorList.java +++ b/javascript/extractor/src/com/semmle/ts/ast/DecoratorList.java @@ -1,26 +1,25 @@ package com.semmle.ts.ast; -import java.util.List; - import com.semmle.js.ast.Decorator; import com.semmle.js.ast.Expression; import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; +import java.util.List; public class DecoratorList extends Expression { - private final List decorators; + private final List decorators; - public DecoratorList(SourceLocation loc, List decorators) { - super("DecoratorList", loc); - this.decorators = decorators; - } + public DecoratorList(SourceLocation loc, List decorators) { + super("DecoratorList", loc); + this.decorators = decorators; + } - public List getDecorators() { - return decorators; - } + public List getDecorators() { + return decorators; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/EnumDeclaration.java b/javascript/extractor/src/com/semmle/ts/ast/EnumDeclaration.java index 2d6c528cb554..6032bdfbf67b 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/EnumDeclaration.java +++ b/javascript/extractor/src/com/semmle/ts/ast/EnumDeclaration.java @@ -1,63 +1,67 @@ package com.semmle.ts.ast; -import java.util.List; - import com.semmle.js.ast.Decorator; import com.semmle.js.ast.Identifier; import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Statement; import com.semmle.js.ast.Visitor; +import java.util.List; public class EnumDeclaration extends Statement implements INodeWithSymbol { - private final boolean isConst; - private final boolean hasDeclareKeyword; - private final List decorators; - private final Identifier id; - private List members; - private int typeSymbol = -1; - - public EnumDeclaration(SourceLocation loc, boolean isConst, boolean hasDeclareKeyword, List decorators, Identifier id, - List members) { - super("EnumDeclaration", loc); - this.isConst = isConst; - this.hasDeclareKeyword = hasDeclareKeyword; - this.decorators = decorators; - this.id = id; - this.members = members; - } - - public boolean isConst() { - return isConst; - } - - public boolean hasDeclareKeyword() { - return hasDeclareKeyword; - } - - public List getDecorators() { - return decorators; - } - - public Identifier getId() { - return id; - } - - public List getMembers() { - return members; - } - - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } - - @Override - public int getSymbol() { - return typeSymbol; - } - - @Override - public void setSymbol(int symbol) { - this.typeSymbol = symbol; - } + private final boolean isConst; + private final boolean hasDeclareKeyword; + private final List decorators; + private final Identifier id; + private List members; + private int typeSymbol = -1; + + public EnumDeclaration( + SourceLocation loc, + boolean isConst, + boolean hasDeclareKeyword, + List decorators, + Identifier id, + List members) { + super("EnumDeclaration", loc); + this.isConst = isConst; + this.hasDeclareKeyword = hasDeclareKeyword; + this.decorators = decorators; + this.id = id; + this.members = members; + } + + public boolean isConst() { + return isConst; + } + + public boolean hasDeclareKeyword() { + return hasDeclareKeyword; + } + + public List getDecorators() { + return decorators; + } + + public Identifier getId() { + return id; + } + + public List getMembers() { + return members; + } + + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } + + @Override + public int getSymbol() { + return typeSymbol; + } + + @Override + public void setSymbol(int symbol) { + this.typeSymbol = symbol; + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/EnumMember.java b/javascript/extractor/src/com/semmle/ts/ast/EnumMember.java index 59ccdf128074..b63fdcf0ff38 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/EnumMember.java +++ b/javascript/extractor/src/com/semmle/ts/ast/EnumMember.java @@ -7,40 +7,40 @@ import com.semmle.js.ast.Visitor; public class EnumMember extends Node implements INodeWithSymbol { - private final Identifier id; - private final Expression initializer; - private int typeSymbol = -1; - - public EnumMember(SourceLocation loc, Identifier id, Expression initializer) { - super("EnumMember", loc); - this.id = id; - this.initializer = initializer; - } - - public Identifier getId() { - return id; - } - - /** - * Returns the initializer expression, or {@code null} if this enum member has - * no explicit initializer. - */ - public Expression getInitializer() { - return initializer; - } - - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } - - @Override - public int getSymbol() { - return typeSymbol; - } - - @Override - public void setSymbol(int symbol) { - this.typeSymbol = symbol; - } + private final Identifier id; + private final Expression initializer; + private int typeSymbol = -1; + + public EnumMember(SourceLocation loc, Identifier id, Expression initializer) { + super("EnumMember", loc); + this.id = id; + this.initializer = initializer; + } + + public Identifier getId() { + return id; + } + + /** + * Returns the initializer expression, or {@code null} if this enum member has no explicit + * initializer. + */ + public Expression getInitializer() { + return initializer; + } + + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } + + @Override + public int getSymbol() { + return typeSymbol; + } + + @Override + public void setSymbol(int symbol) { + this.typeSymbol = symbol; + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/ExportAsNamespaceDeclaration.java b/javascript/extractor/src/com/semmle/ts/ast/ExportAsNamespaceDeclaration.java index 19d0b612fbb9..b6921b34924f 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/ExportAsNamespaceDeclaration.java +++ b/javascript/extractor/src/com/semmle/ts/ast/ExportAsNamespaceDeclaration.java @@ -5,24 +5,21 @@ import com.semmle.js.ast.Statement; import com.semmle.js.ast.Visitor; -/** - * A statement of form export as namespace X where X is an - * identifier. - */ +/** A statement of form export as namespace X where X is an identifier. */ public class ExportAsNamespaceDeclaration extends Statement { - private Identifier id; + private Identifier id; - public ExportAsNamespaceDeclaration(SourceLocation loc, Identifier id) { - super("ExportAsNamespaceDeclaration", loc); - this.id = id; - } + public ExportAsNamespaceDeclaration(SourceLocation loc, Identifier id) { + super("ExportAsNamespaceDeclaration", loc); + this.id = id; + } - public Identifier getId() { - return id; - } + public Identifier getId() { + return id; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/ExportWholeDeclaration.java b/javascript/extractor/src/com/semmle/ts/ast/ExportWholeDeclaration.java index 5e5ce4308c8f..1f35b95d269a 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/ExportWholeDeclaration.java +++ b/javascript/extractor/src/com/semmle/ts/ast/ExportWholeDeclaration.java @@ -6,19 +6,19 @@ import com.semmle.js.ast.Visitor; public class ExportWholeDeclaration extends Statement { - private final Expression rhs; + private final Expression rhs; - public ExportWholeDeclaration(SourceLocation loc, Expression rhs) { - super("ExportWholeDeclaration", loc); - this.rhs = rhs; - } + public ExportWholeDeclaration(SourceLocation loc, Expression rhs) { + super("ExportWholeDeclaration", loc); + this.rhs = rhs; + } - public Expression getRhs() { - return rhs; - } + public Expression getRhs() { + return rhs; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/ExpressionWithTypeArguments.java b/javascript/extractor/src/com/semmle/ts/ast/ExpressionWithTypeArguments.java index 117d4cbf6699..4c4c681d0784 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/ExpressionWithTypeArguments.java +++ b/javascript/extractor/src/com/semmle/ts/ast/ExpressionWithTypeArguments.java @@ -1,42 +1,41 @@ package com.semmle.ts.ast; -import java.util.List; - import com.semmle.js.ast.Expression; import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; +import java.util.List; /** - * An expression with type arguments, occurring as the super-class expression of - * a class. For example: + * An expression with type arguments, occurring as the super-class expression of a class. For + * example: * * 

        * class StringList extends List<string> {}
  *
      * - * Above, List is a concrete expression whereas its type argument is a - * type. + * Above, List is a concrete expression whereas its type argument is a type. */ public class ExpressionWithTypeArguments extends Expression { - private final Expression expression; - private final List typeArguments; + private final Expression expression; + private final List typeArguments; - public ExpressionWithTypeArguments(SourceLocation loc, Expression expression, List typeArguments) { - super("ExpressionWithTypeArguments", loc); - this.expression = expression; - this.typeArguments = typeArguments; - } + public ExpressionWithTypeArguments( + SourceLocation loc, Expression expression, List typeArguments) { + super("ExpressionWithTypeArguments", loc); + this.expression = expression; + this.typeArguments = typeArguments; + } - public Expression getExpression() { - return expression; - } + public Expression getExpression() { + return expression; + } - public List getTypeArguments() { - return typeArguments; - } + public List getTypeArguments() { + return typeArguments; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/ExternalModuleDeclaration.java b/javascript/extractor/src/com/semmle/ts/ast/ExternalModuleDeclaration.java index 15d04ae2f7a7..58baaac74245 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/ExternalModuleDeclaration.java +++ b/javascript/extractor/src/com/semmle/ts/ast/ExternalModuleDeclaration.java @@ -1,35 +1,32 @@ package com.semmle.ts.ast; -import java.util.List; - import com.semmle.js.ast.Literal; import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Statement; import com.semmle.js.ast.Visitor; +import java.util.List; -/** - * A statement of form declare module "X" {...}. - */ +/** A statement of form declare module "X" {...}. */ public class ExternalModuleDeclaration extends Statement { - private final Literal name; - private final List body; + private final Literal name; + private final List body; - public ExternalModuleDeclaration(SourceLocation loc, Literal name, List body) { - super("ExternalModuleDeclaration", loc); - this.name = name; - this.body = body; - } + public ExternalModuleDeclaration(SourceLocation loc, Literal name, List body) { + super("ExternalModuleDeclaration", loc); + this.name = name; + this.body = body; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } - public Literal getName() { - return name; - } + public Literal getName() { + return name; + } - public List getBody() { - return body; - } + public List getBody() { + return body; + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/ExternalModuleReference.java b/javascript/extractor/src/com/semmle/ts/ast/ExternalModuleReference.java index 8819efc52160..48b03ae5a164 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/ExternalModuleReference.java +++ b/javascript/extractor/src/com/semmle/ts/ast/ExternalModuleReference.java @@ -5,20 +5,19 @@ import com.semmle.js.ast.Visitor; public class ExternalModuleReference extends Expression { - private final Expression expression; + private final Expression expression; - public ExternalModuleReference(SourceLocation loc, Expression expression) { - super("ExternalModuleReference", loc); - this.expression = expression; - } + public ExternalModuleReference(SourceLocation loc, Expression expression) { + super("ExternalModuleReference", loc); + this.expression = expression; + } - public Expression getExpression() { - return expression; - } - - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + public Expression getExpression() { + return expression; + } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/FunctionTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/FunctionTypeExpr.java index 67556f20de2b..b4015b123775 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/FunctionTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/FunctionTypeExpr.java @@ -5,25 +5,25 @@ import com.semmle.js.ast.Visitor; public class FunctionTypeExpr extends TypeExpression { - private final FunctionExpression function; - private final boolean isConstructor; + private final FunctionExpression function; + private final boolean isConstructor; - public FunctionTypeExpr(SourceLocation loc, FunctionExpression function, boolean isConstructor) { - super("FunctionTypeExpr", loc); - this.function = function; - this.isConstructor = isConstructor; - } + public FunctionTypeExpr(SourceLocation loc, FunctionExpression function, boolean isConstructor) { + super("FunctionTypeExpr", loc); + this.function = function; + this.isConstructor = isConstructor; + } - public FunctionExpression getFunction() { - return function; - } + public FunctionExpression getFunction() { + return function; + } - public boolean isConstructor() { - return isConstructor; - } + public boolean isConstructor() { + return isConstructor; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/GenericTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/GenericTypeExpr.java index f7945a4e8897..efaee334c1fb 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/GenericTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/GenericTypeExpr.java @@ -1,33 +1,31 @@ package com.semmle.ts.ast; -import java.util.List; - import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; +import java.util.List; -/** - * An instantiation of a named type, such as Array<number> - */ +/** An instantiation of a named type, such as Array<number> */ public class GenericTypeExpr extends TypeExpression { - final private ITypeExpression typeName; // Always Identifier or MemberExpression - final private List typeArguments; + private final ITypeExpression typeName; // Always Identifier or MemberExpression + private final List typeArguments; - public GenericTypeExpr(SourceLocation loc, ITypeExpression typeName, List typeArguments) { - super("GenericTypeExpr", loc); - this.typeName = typeName; - this.typeArguments = typeArguments; - } + public GenericTypeExpr( + SourceLocation loc, ITypeExpression typeName, List typeArguments) { + super("GenericTypeExpr", loc); + this.typeName = typeName; + this.typeArguments = typeArguments; + } - public ITypeExpression getTypeName() { - return typeName; - } + public ITypeExpression getTypeName() { + return typeName; + } - public List getTypeArguments() { - return typeArguments; - } + public List getTypeArguments() { + return typeArguments; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/GlobalAugmentationDeclaration.java b/javascript/extractor/src/com/semmle/ts/ast/GlobalAugmentationDeclaration.java index dc2ec3a3e57e..b8eec7fac94b 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/GlobalAugmentationDeclaration.java +++ b/javascript/extractor/src/com/semmle/ts/ast/GlobalAugmentationDeclaration.java @@ -1,28 +1,25 @@ package com.semmle.ts.ast; -import java.util.List; - import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Statement; import com.semmle.js.ast.Visitor; +import java.util.List; -/** - * A statement of form: declare global { ... } - */ +/** A statement of form: declare global { ... } */ public class GlobalAugmentationDeclaration extends Statement { - private final List body; + private final List body; - public GlobalAugmentationDeclaration(SourceLocation loc, List body) { - super("GlobalAugmentationDeclaration", loc); - this.body = body; - } + public GlobalAugmentationDeclaration(SourceLocation loc, List body) { + super("GlobalAugmentationDeclaration", loc); + this.body = body; + } - public List getBody() { - return body; - } + public List getBody() { + return body; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/INodeWithSymbol.java b/javascript/extractor/src/com/semmle/ts/ast/INodeWithSymbol.java index 858f71b0a398..9f5e1fe29937 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/INodeWithSymbol.java +++ b/javascript/extractor/src/com/semmle/ts/ast/INodeWithSymbol.java @@ -3,15 +3,15 @@ /** * An AST node that is associated with a TypeScript compiler symbol. * - * This can be the symbol for the type defined by this node, of it this is a - * module top-level, the symbol for that module. + *

      This can be the symbol for the type defined by this node, of it this is a module top-level, + * the symbol for that module. */ public interface INodeWithSymbol { - /** - * Gets a number identifying the symbol associated with this AST node, or - * -1 if there is no such symbol. - */ - int getSymbol(); + /** + * Gets a number identifying the symbol associated with this AST node, or -1 if there is + * no such symbol. + */ + int getSymbol(); - void setSymbol(int symbol); + void setSymbol(int symbol); } diff --git a/javascript/extractor/src/com/semmle/ts/ast/ITypeExpression.java b/javascript/extractor/src/com/semmle/ts/ast/ITypeExpression.java index 2f711f449d6f..0054adb4f72f 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/ITypeExpression.java +++ b/javascript/extractor/src/com/semmle/ts/ast/ITypeExpression.java @@ -5,11 +5,9 @@ /** * An AST node that may occur as part of a TypeScript type annotation. - *

      - * At the QL level, expressions and type annotations are completely separate. In - * the extractor, however, some expressions such as {@link Literal} type may - * occur in a type annotation because the TypeScript AST does not distinguish - * null literals from the null type. + * + *

      At the QL level, expressions and type annotations are completely separate. In the extractor, + * however, some expressions such as {@link Literal} type may occur in a type annotation because the + * TypeScript AST does not distinguish null literals from the null type. */ -public interface ITypeExpression extends INode, ITypedAstNode { -} +public interface ITypeExpression extends INode, ITypedAstNode {} diff --git a/javascript/extractor/src/com/semmle/ts/ast/ITypedAstNode.java b/javascript/extractor/src/com/semmle/ts/ast/ITypedAstNode.java index b360d292c861..e2d20a41fb33 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/ITypedAstNode.java +++ b/javascript/extractor/src/com/semmle/ts/ast/ITypedAstNode.java @@ -1,17 +1,15 @@ package com.semmle.ts.ast; -/** - * An AST node with an associated static type. - */ +/** An AST node with an associated static type. */ public interface ITypedAstNode { - /** - * Gets the static type of this node as determined by the TypeScript compiler, - * or -1 if no type was determined. - *

      - * The ID refers to a type in a table that is extracted on a per-project basis, - * and the meaning of this type ID is not available at the AST level. - */ - int getStaticTypeId(); + /** + * Gets the static type of this node as determined by the TypeScript compiler, or -1 if no type + * was determined. + * + *

      The ID refers to a type in a table that is extracted on a per-project basis, and the meaning + * of this type ID is not available at the AST level. + */ + int getStaticTypeId(); - void setStaticTypeId(int id); + void setStaticTypeId(int id); } diff --git a/javascript/extractor/src/com/semmle/ts/ast/ImportTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/ImportTypeExpr.java index 4f2e14522d03..edf60004645c 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/ImportTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/ImportTypeExpr.java @@ -4,23 +4,21 @@ import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; -/** - * An import type such as in import("http").ServerRequest. - */ +/** An import type such as in import("http").ServerRequest. */ public class ImportTypeExpr extends Expression implements ITypeExpression { - private final ITypeExpression path; + private final ITypeExpression path; - public ImportTypeExpr(SourceLocation loc, ITypeExpression path) { - super("ImportTypeExpr", loc); - this.path = path; - } + public ImportTypeExpr(SourceLocation loc, ITypeExpression path) { + super("ImportTypeExpr", loc); + this.path = path; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } - public ITypeExpression getPath() { - return path; - } + public ITypeExpression getPath() { + return path; + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/ImportWholeDeclaration.java b/javascript/extractor/src/com/semmle/ts/ast/ImportWholeDeclaration.java index 81745f1158cc..4631a615fe2b 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/ImportWholeDeclaration.java +++ b/javascript/extractor/src/com/semmle/ts/ast/ImportWholeDeclaration.java @@ -6,29 +6,27 @@ import com.semmle.js.ast.Statement; import com.semmle.js.ast.Visitor; -/** - * An import of form import a = E. - */ +/** An import of form import a = E. */ public class ImportWholeDeclaration extends Statement { - private final Identifier lhs; - private final Expression rhs; + private final Identifier lhs; + private final Expression rhs; - public ImportWholeDeclaration(SourceLocation loc, Identifier lhs, Expression rhs) { - super("ImportWholeDeclaration", loc); - this.lhs = lhs; - this.rhs = rhs; - } + public ImportWholeDeclaration(SourceLocation loc, Identifier lhs, Expression rhs) { + super("ImportWholeDeclaration", loc); + this.lhs = lhs; + this.rhs = rhs; + } - public Identifier getLhs() { - return lhs; - } + public Identifier getLhs() { + return lhs; + } - public Expression getRhs() { - return rhs; - } + public Expression getRhs() { + return rhs; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/IndexedAccessTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/IndexedAccessTypeExpr.java index d6f723da2e8f..baf7b95ff2e1 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/IndexedAccessTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/IndexedAccessTypeExpr.java @@ -3,29 +3,28 @@ import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; -/** - * A type of form T[K] where T and K are types. - */ +/** A type of form T[K] where T and K are types. */ public class IndexedAccessTypeExpr extends TypeExpression { - private final ITypeExpression objectType; - private final ITypeExpression indexType; + private final ITypeExpression objectType; + private final ITypeExpression indexType; - public IndexedAccessTypeExpr(SourceLocation loc, ITypeExpression objectType, ITypeExpression indexType) { - super("IndexedAccessTypeExpr", loc); - this.objectType = objectType; - this.indexType = indexType; - } + public IndexedAccessTypeExpr( + SourceLocation loc, ITypeExpression objectType, ITypeExpression indexType) { + super("IndexedAccessTypeExpr", loc); + this.objectType = objectType; + this.indexType = indexType; + } - public ITypeExpression getObjectType() { - return objectType; - } + public ITypeExpression getObjectType() { + return objectType; + } - public ITypeExpression getIndexType() { - return indexType; - } + public ITypeExpression getIndexType() { + return indexType; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/InferTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/InferTypeExpr.java index be59885e1833..a18269ce6c2c 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/InferTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/InferTypeExpr.java @@ -3,23 +3,21 @@ import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; -/** - * A type annotation of form infer R - */ +/** A type annotation of form infer R */ public class InferTypeExpr extends TypeExpression { - private TypeParameter typeParameter; + private TypeParameter typeParameter; - public InferTypeExpr(SourceLocation loc, TypeParameter typeParameter) { - super("InferTypeExpr", loc); - this.typeParameter = typeParameter; - } + public InferTypeExpr(SourceLocation loc, TypeParameter typeParameter) { + super("InferTypeExpr", loc); + this.typeParameter = typeParameter; + } - public TypeParameter getTypeParameter() { - return typeParameter; - } + public TypeParameter getTypeParameter() { + return typeParameter; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/InterfaceDeclaration.java b/javascript/extractor/src/com/semmle/ts/ast/InterfaceDeclaration.java index 6ecb30a44c96..546790c82303 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/InterfaceDeclaration.java +++ b/javascript/extractor/src/com/semmle/ts/ast/InterfaceDeclaration.java @@ -1,65 +1,65 @@ package com.semmle.ts.ast; -import java.util.List; - import com.semmle.js.ast.Identifier; import com.semmle.js.ast.MemberDefinition; import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Statement; import com.semmle.js.ast.Visitor; +import java.util.List; -/** - * A TypeScript interface declaration. - */ +/** A TypeScript interface declaration. */ public class InterfaceDeclaration extends Statement implements INodeWithSymbol { - private final Identifier name; - private final List typeParameters; - private final List superInterfaces; - private final List> body; - private int typeSymbol = -1; + private final Identifier name; + private final List typeParameters; + private final List superInterfaces; + private final List> body; + private int typeSymbol = -1; - public InterfaceDeclaration(SourceLocation loc, Identifier name, List typeParameters, - List superInterfaces, - List> body) { - super("InterfaceDeclaration", loc); - this.name = name; - this.typeParameters = typeParameters; - this.superInterfaces = superInterfaces; - this.body = body; - } + public InterfaceDeclaration( + SourceLocation loc, + Identifier name, + List typeParameters, + List superInterfaces, + List> body) { + super("InterfaceDeclaration", loc); + this.name = name; + this.typeParameters = typeParameters; + this.superInterfaces = superInterfaces; + this.body = body; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } - public Identifier getName() { - return name; - } + public Identifier getName() { + return name; + } - public List getTypeParameters() { - return typeParameters; - } + public List getTypeParameters() { + return typeParameters; + } - public boolean hasTypeParameters() { - return !typeParameters.isEmpty(); - } + public boolean hasTypeParameters() { + return !typeParameters.isEmpty(); + } - public List> getBody() { - return body; - } + public List> getBody() { + return body; + } - public List getSuperInterfaces() { - return superInterfaces; - } + public List getSuperInterfaces() { + return superInterfaces; + } - @Override - public int getSymbol() { - return typeSymbol; - } + @Override + public int getSymbol() { + return typeSymbol; + } - @Override - public void setSymbol(int symbol) { - this.typeSymbol = symbol; - } + @Override + public void setSymbol(int symbol) { + this.typeSymbol = symbol; + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/InterfaceTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/InterfaceTypeExpr.java index 09a38c869b34..20f2b45b2b82 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/InterfaceTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/InterfaceTypeExpr.java @@ -1,28 +1,25 @@ package com.semmle.ts.ast; -import java.util.List; - import com.semmle.js.ast.MemberDefinition; import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; +import java.util.List; -/** - * An inline interface type, such as {x: number; y: number}. - */ +/** An inline interface type, such as {x: number; y: number}. */ public class InterfaceTypeExpr extends TypeExpression { - private final List> body; + private final List> body; - public InterfaceTypeExpr(SourceLocation loc, List> body) { - super("InterfaceTypeExpr", loc); - this.body = body; - } + public InterfaceTypeExpr(SourceLocation loc, List> body) { + super("InterfaceTypeExpr", loc); + this.body = body; + } - public List> getBody() { - return body; - } + public List> getBody() { + return body; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/IntersectionTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/IntersectionTypeExpr.java index 05a794908ea3..2ff10be12cb5 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/IntersectionTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/IntersectionTypeExpr.java @@ -1,29 +1,28 @@ package com.semmle.ts.ast; -import java.util.List; - import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; +import java.util.List; /** - * An intersection type such as T&S, denoting the intersection of - * type T and type S. + * An intersection type such as T&S, denoting the intersection of type T and + * type S. */ public class IntersectionTypeExpr extends TypeExpression { - private final List elementTypes; + private final List elementTypes; - public IntersectionTypeExpr(SourceLocation loc, List elementTypes) { - super("IntersectionTypeExpr", loc); - this.elementTypes = elementTypes; - } + public IntersectionTypeExpr(SourceLocation loc, List elementTypes) { + super("IntersectionTypeExpr", loc); + this.elementTypes = elementTypes; + } - /** The members of the intersection type; always contains at least two types. */ - public List getElementTypes() { - return elementTypes; - } + /** The members of the intersection type; always contains at least two types. */ + public List getElementTypes() { + return elementTypes; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/IsTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/IsTypeExpr.java index 69b7d67dd166..a64fdb83650a 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/IsTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/IsTypeExpr.java @@ -4,29 +4,29 @@ import com.semmle.js.ast.Visitor; /** - * A type of form E is T where E is a parameter name or - * this and T is a type. + * A type of form E is T where E is a parameter name or this and + * T is a type. */ public class IsTypeExpr extends TypeExpression { - private final ITypeExpression left; // Always Identifier or KeywordTypeExpr (in case of 'this') - private final ITypeExpression right; + private final ITypeExpression left; // Always Identifier or KeywordTypeExpr (in case of 'this') + private final ITypeExpression right; - public IsTypeExpr(SourceLocation loc, ITypeExpression left, ITypeExpression right) { - super("IsTypeExpr", loc); - this.left = left; - this.right = right; - } + public IsTypeExpr(SourceLocation loc, ITypeExpression left, ITypeExpression right) { + super("IsTypeExpr", loc); + this.left = left; + this.right = right; + } - public ITypeExpression getLeft() { - return left; - } + public ITypeExpression getLeft() { + return left; + } - public ITypeExpression getRight() { - return right; - } + public ITypeExpression getRight() { + return right; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/KeyofTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/KeyofTypeExpr.java index ecd2e51d6313..a65a4b0ee44b 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/KeyofTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/KeyofTypeExpr.java @@ -3,23 +3,21 @@ import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; -/** - * A type of form keyof T where T is a type. - */ +/** A type of form keyof T where T is a type. */ public class KeyofTypeExpr extends TypeExpression { - private final ITypeExpression elementType; + private final ITypeExpression elementType; - public KeyofTypeExpr(SourceLocation loc, ITypeExpression elementType) { - super("KeyofTypeExpr", loc); - this.elementType = elementType; - } + public KeyofTypeExpr(SourceLocation loc, ITypeExpression elementType) { + super("KeyofTypeExpr", loc); + this.elementType = elementType; + } - public ITypeExpression getElementType() { - return elementType; - } + public ITypeExpression getElementType() { + return elementType; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/KeywordTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/KeywordTypeExpr.java index de0ab2a82bd2..ede17e079e9b 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/KeywordTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/KeywordTypeExpr.java @@ -5,28 +5,28 @@ /** * One of the TypeScript keyword types, such as string or any. - *

      - * This includes the type unique symbol which consists of two keywords - * but is represented as a keyword single type expression. - *

      - * At the QL level, the null type is also a keyword type. In the - * extractor, however, this is represented by a Literal, because the TypeScript - * AST does not distinguish those two uses of null. + * + *

      This includes the type unique symbol which consists of two keywords but is + * represented as a keyword single type expression. + * + *

      At the QL level, the null type is also a keyword type. In the extractor, however, + * this is represented by a Literal, because the TypeScript AST does not distinguish those two uses + * of null. */ public class KeywordTypeExpr extends TypeExpression { - private final String keyword; + private final String keyword; - public KeywordTypeExpr(SourceLocation loc, String keyword) { - super("KeywordTypeExpr", loc); - this.keyword = keyword; - } + public KeywordTypeExpr(SourceLocation loc, String keyword) { + super("KeywordTypeExpr", loc); + this.keyword = keyword; + } - public String getKeyword() { - return keyword; - } + public String getKeyword() { + return keyword; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/MappedTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/MappedTypeExpr.java index 0744ef3eaef5..30129296f348 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/MappedTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/MappedTypeExpr.java @@ -4,32 +4,32 @@ import com.semmle.js.ast.Visitor; /** - * A type of form { [K in C]: T }, where T is a type that may - * refer to K. - *

      - * As with the TypeScript AST, the K in C part is represented as a type - * parameter with C as its upper bound. + * A type of form { [K in C]: T }, where T is a type that may refer to K. + * + *

      As with the TypeScript AST, the K in C part is represented as a type parameter with + * C as its upper bound. */ public class MappedTypeExpr extends TypeExpression { - private final TypeParameter typeParameter; - private final ITypeExpression elementType; + private final TypeParameter typeParameter; + private final ITypeExpression elementType; - public MappedTypeExpr(SourceLocation loc, TypeParameter typeParameter, ITypeExpression elementType) { - super("MappedTypeExpr", loc); - this.typeParameter = typeParameter; - this.elementType = elementType; - } + public MappedTypeExpr( + SourceLocation loc, TypeParameter typeParameter, ITypeExpression elementType) { + super("MappedTypeExpr", loc); + this.typeParameter = typeParameter; + this.elementType = elementType; + } - public TypeParameter getTypeParameter() { - return typeParameter; - } + public TypeParameter getTypeParameter() { + return typeParameter; + } - public ITypeExpression getElementType() { - return elementType; - } + public ITypeExpression getElementType() { + return elementType; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/NamespaceDeclaration.java b/javascript/extractor/src/com/semmle/ts/ast/NamespaceDeclaration.java index a36d6addd439..c859a7310520 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/NamespaceDeclaration.java +++ b/javascript/extractor/src/com/semmle/ts/ast/NamespaceDeclaration.java @@ -1,62 +1,66 @@ package com.semmle.ts.ast; -import java.util.List; - import com.semmle.js.ast.Identifier; import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Statement; import com.semmle.js.ast.Visitor; +import java.util.List; public class NamespaceDeclaration extends Statement implements INodeWithSymbol { - private final Identifier name; - private final List body; - private final boolean isInstantiated; - private final boolean hasDeclareKeyword; - private int symbol = -1; - - public NamespaceDeclaration(SourceLocation loc, Identifier name, List body, boolean isInstantiated, boolean hasDeclareKeyword) { - super("NamespaceDeclaration", loc); - this.name = name; - this.body = body; - this.isInstantiated = isInstantiated; - this.hasDeclareKeyword = hasDeclareKeyword; - } - - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } - - public Identifier getName() { - return name; - } - - public List getBody() { - return body; - } - - /** - * Returns whether this is an instantiated namespace. - * - * Non-instantiated namespaces only contain interface types, type aliases, and - * other non-instantiated namespaces. The TypeScript compiler does not emit - * code for non-instantiated namespaces. - */ - public boolean isInstantiated() { - return isInstantiated; - } - - public boolean hasDeclareKeyword() { - return hasDeclareKeyword; - } - - @Override - public int getSymbol() { - return this.symbol; - } - - @Override - public void setSymbol(int symbol) { - this.symbol = symbol; - } + private final Identifier name; + private final List body; + private final boolean isInstantiated; + private final boolean hasDeclareKeyword; + private int symbol = -1; + + public NamespaceDeclaration( + SourceLocation loc, + Identifier name, + List body, + boolean isInstantiated, + boolean hasDeclareKeyword) { + super("NamespaceDeclaration", loc); + this.name = name; + this.body = body; + this.isInstantiated = isInstantiated; + this.hasDeclareKeyword = hasDeclareKeyword; + } + + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } + + public Identifier getName() { + return name; + } + + public List getBody() { + return body; + } + + /** + * Returns whether this is an instantiated namespace. + * + *

      Non-instantiated namespaces only contain interface types, type aliases, and other + * non-instantiated namespaces. The TypeScript compiler does not emit code for non-instantiated + * namespaces. + */ + public boolean isInstantiated() { + return isInstantiated; + } + + public boolean hasDeclareKeyword() { + return hasDeclareKeyword; + } + + @Override + public int getSymbol() { + return this.symbol; + } + + @Override + public void setSymbol(int symbol) { + this.symbol = symbol; + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/NonNullAssertion.java b/javascript/extractor/src/com/semmle/ts/ast/NonNullAssertion.java index 3cb53f4e59a7..485b827ce976 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/NonNullAssertion.java +++ b/javascript/extractor/src/com/semmle/ts/ast/NonNullAssertion.java @@ -4,23 +4,21 @@ import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; -/** - * A TypeScript expression of form E!, asserting that E is not null. - */ +/** A TypeScript expression of form E!, asserting that E is not null. */ public class NonNullAssertion extends Expression { - private final Expression expression; + private final Expression expression; - public NonNullAssertion(SourceLocation loc, Expression expression) { - super("NonNullAssertion", loc); - this.expression = expression; - } + public NonNullAssertion(SourceLocation loc, Expression expression) { + super("NonNullAssertion", loc); + this.expression = expression; + } - public Expression getExpression() { - return expression; - } + public Expression getExpression() { + return expression; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/OptionalTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/OptionalTypeExpr.java index 483f38709125..3d98f563127e 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/OptionalTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/OptionalTypeExpr.java @@ -3,24 +3,21 @@ import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; -/** - * An optional type in a tuple type, such as number? in - * [string, number?]. - */ +/** An optional type in a tuple type, such as number? in [string, number?]. */ public class OptionalTypeExpr extends TypeExpression { - private final ITypeExpression elementType; + private final ITypeExpression elementType; - public OptionalTypeExpr(SourceLocation loc, ITypeExpression elementType) { - super("OptionalTypeExpr", loc); - this.elementType = elementType; - } + public OptionalTypeExpr(SourceLocation loc, ITypeExpression elementType) { + super("OptionalTypeExpr", loc); + this.elementType = elementType; + } - public ITypeExpression getElementType() { - return elementType; - } + public ITypeExpression getElementType() { + return elementType; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/ParenthesizedTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/ParenthesizedTypeExpr.java index f9fc1fe6bb0d..1be6f191ff79 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/ParenthesizedTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/ParenthesizedTypeExpr.java @@ -3,23 +3,21 @@ import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; -/** - * A type expression in parentheses, such as ("foo" | "bar"). - */ +/** A type expression in parentheses, such as ("foo" | "bar"). */ public class ParenthesizedTypeExpr extends TypeExpression { - private final ITypeExpression elementType; + private final ITypeExpression elementType; - public ParenthesizedTypeExpr(SourceLocation loc, ITypeExpression elementType) { - super("ParenthesizedTypeExpr", loc); - this.elementType = elementType; - } + public ParenthesizedTypeExpr(SourceLocation loc, ITypeExpression elementType) { + super("ParenthesizedTypeExpr", loc); + this.elementType = elementType; + } - public ITypeExpression getElementType() { - return elementType; - } + public ITypeExpression getElementType() { + return elementType; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/RestTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/RestTypeExpr.java index 1c0618eea9e4..ef0d9ad7f7f9 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/RestTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/RestTypeExpr.java @@ -3,24 +3,21 @@ import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; -/** - * A rest type in a tuple type, such as number[] in - * [string, ...number[]]. - */ +/** A rest type in a tuple type, such as number[] in [string, ...number[]]. */ public class RestTypeExpr extends TypeExpression { - private final ITypeExpression arrayType; + private final ITypeExpression arrayType; - public RestTypeExpr(SourceLocation loc, ITypeExpression arrayType) { - super("RestTypeExpr", loc); - this.arrayType = arrayType; - } + public RestTypeExpr(SourceLocation loc, ITypeExpression arrayType) { + super("RestTypeExpr", loc); + this.arrayType = arrayType; + } - public ITypeExpression getArrayType() { - return arrayType; - } + public ITypeExpression getArrayType() { + return arrayType; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/TupleTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/TupleTypeExpr.java index 6f8b0a7a8e65..5b9044cd63fc 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/TupleTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/TupleTypeExpr.java @@ -1,27 +1,24 @@ package com.semmle.ts.ast; -import java.util.List; - import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; +import java.util.List; -/** - * A tuple type, such as [number, string]. - */ +/** A tuple type, such as [number, string]. */ public class TupleTypeExpr extends TypeExpression { - private final List elementTypes; + private final List elementTypes; - public TupleTypeExpr(SourceLocation loc, List elementTypes) { - super("TupleTypeExpr", loc); - this.elementTypes = elementTypes; - } + public TupleTypeExpr(SourceLocation loc, List elementTypes) { + super("TupleTypeExpr", loc); + this.elementTypes = elementTypes; + } - public List getElementTypes() { - return elementTypes; - } + public List getElementTypes() { + return elementTypes; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/TypeAliasDeclaration.java b/javascript/extractor/src/com/semmle/ts/ast/TypeAliasDeclaration.java index 2783d7a09a8f..98ac24fc9d5d 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/TypeAliasDeclaration.java +++ b/javascript/extractor/src/com/semmle/ts/ast/TypeAliasDeclaration.java @@ -1,53 +1,56 @@ package com.semmle.ts.ast; -import java.util.List; - import com.semmle.js.ast.Identifier; import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Statement; import com.semmle.js.ast.Visitor; +import java.util.List; public class TypeAliasDeclaration extends Statement implements INodeWithSymbol { - private final Identifier name; - private final List typeParameters; - private final ITypeExpression definition; - private int typeSymbol = -1; - - public TypeAliasDeclaration(SourceLocation loc, Identifier name, List typeParameters, ITypeExpression definition) { - super("TypeAliasDeclaration", loc); - this.name = name; - this.typeParameters = typeParameters; - this.definition = definition; - } - - public Identifier getId() { - return name; - } - - public List getTypeParameters() { - return typeParameters; - } - - public boolean hasTypeParameters() { - return !typeParameters.isEmpty(); - } - - public ITypeExpression getDefinition() { - return definition; - } - - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } - - @Override - public int getSymbol() { - return typeSymbol; - } - - @Override - public void setSymbol(int symbol) { - this.typeSymbol = symbol; - } + private final Identifier name; + private final List typeParameters; + private final ITypeExpression definition; + private int typeSymbol = -1; + + public TypeAliasDeclaration( + SourceLocation loc, + Identifier name, + List typeParameters, + ITypeExpression definition) { + super("TypeAliasDeclaration", loc); + this.name = name; + this.typeParameters = typeParameters; + this.definition = definition; + } + + public Identifier getId() { + return name; + } + + public List getTypeParameters() { + return typeParameters; + } + + public boolean hasTypeParameters() { + return !typeParameters.isEmpty(); + } + + public ITypeExpression getDefinition() { + return definition; + } + + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } + + @Override + public int getSymbol() { + return typeSymbol; + } + + @Override + public void setSymbol(int symbol) { + this.typeSymbol = symbol; + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/TypeAssertion.java b/javascript/extractor/src/com/semmle/ts/ast/TypeAssertion.java index 116c9cbfa22b..6a9150f93c29 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/TypeAssertion.java +++ b/javascript/extractor/src/com/semmle/ts/ast/TypeAssertion.java @@ -4,39 +4,41 @@ import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; -/** - * An expression of form E as T or <T> E. - */ +/** An expression of form E as T or <T> E. */ public class TypeAssertion extends Expression { - private final Expression expression; - private final ITypeExpression typeAnnotation; - private final boolean isAsExpression; + private final Expression expression; + private final ITypeExpression typeAnnotation; + private final boolean isAsExpression; - public TypeAssertion(SourceLocation loc, Expression expression, ITypeExpression typeAnnotation, boolean isAsExpression) { - super("TypeAssertion", loc); - this.expression = expression; - this.typeAnnotation = typeAnnotation; - this.isAsExpression = isAsExpression; - } + public TypeAssertion( + SourceLocation loc, + Expression expression, + ITypeExpression typeAnnotation, + boolean isAsExpression) { + super("TypeAssertion", loc); + this.expression = expression; + this.typeAnnotation = typeAnnotation; + this.isAsExpression = isAsExpression; + } - public Expression getExpression() { - return expression; - } + public Expression getExpression() { + return expression; + } - public ITypeExpression getTypeAnnotation() { - return typeAnnotation; - } + public ITypeExpression getTypeAnnotation() { + return typeAnnotation; + } - /** - * True if this is an assertion of form E as T, as opposed to the old - * syntax <T> E. - */ - public boolean isAsExpression() { - return isAsExpression; - } + /** + * True if this is an assertion of form E as T, as opposed to the old syntax + * <T> E. + */ + public boolean isAsExpression() { + return isAsExpression; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/TypeExpression.java b/javascript/extractor/src/com/semmle/ts/ast/TypeExpression.java index a662810a7168..1cc562faa174 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/TypeExpression.java +++ b/javascript/extractor/src/com/semmle/ts/ast/TypeExpression.java @@ -4,23 +4,22 @@ import com.semmle.js.ast.SourceLocation; /** - * An AST node that may occur as part of a TypeScript type annotation and is not - * also an expression. + * An AST node that may occur as part of a TypeScript type annotation and is not also an expression. */ public abstract class TypeExpression extends Node implements ITypeExpression { - private int staticTypeId = -1; + private int staticTypeId = -1; - public TypeExpression(String type, SourceLocation loc) { - super(type, loc); - } + public TypeExpression(String type, SourceLocation loc) { + super(type, loc); + } - @Override - public int getStaticTypeId() { - return staticTypeId; - } + @Override + public int getStaticTypeId() { + return staticTypeId; + } - @Override - public void setStaticTypeId(int staticTypeId) { - this.staticTypeId = staticTypeId; - } + @Override + public void setStaticTypeId(int staticTypeId) { + this.staticTypeId = staticTypeId; + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/TypeParameter.java b/javascript/extractor/src/com/semmle/ts/ast/TypeParameter.java index 530edb3ab54a..7cbf8d2d0eb2 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/TypeParameter.java +++ b/javascript/extractor/src/com/semmle/ts/ast/TypeParameter.java @@ -6,48 +6,46 @@ /** * A type parameter declared on a class, interface, function, or type alias. - *

      - * The general form of a type parameter is: S extends T = U. + * + *

      The general form of a type parameter is: S extends T = U. */ public class TypeParameter extends TypeExpression { - private final Identifier id; - private final ITypeExpression bound; - private final ITypeExpression default_; + private final Identifier id; + private final ITypeExpression bound; + private final ITypeExpression default_; - public TypeParameter(SourceLocation loc, Identifier id, ITypeExpression bound, ITypeExpression default_) { - super("TypeParameter", loc); - this.id = id; - this.bound = bound; - this.default_ = default_; - } + public TypeParameter( + SourceLocation loc, Identifier id, ITypeExpression bound, ITypeExpression default_) { + super("TypeParameter", loc); + this.id = id; + this.bound = bound; + this.default_ = default_; + } - public Identifier getId() { - return id; - } + public Identifier getId() { + return id; + } - /** - * Returns the bound on the type parameter, or {@code null} if there is no - * bound. - *

      - * For example, in T extends Array = number[] the bound is - * Array. - */ - public ITypeExpression getBound() { - return bound; - } + /** + * Returns the bound on the type parameter, or {@code null} if there is no bound. + * + *

      For example, in T extends Array = number[] the bound is Array. + */ + public ITypeExpression getBound() { + return bound; + } - /** - * Returns the type parameter default, or {@code null} if there is no default, - *

      - * For example, in T extends Array = number[] the default is - * number[]. - */ - public ITypeExpression getDefault() { - return default_; - } + /** + * Returns the type parameter default, or {@code null} if there is no default, + * + *

      For example, in T extends Array = number[] the default is number[]. + */ + public ITypeExpression getDefault() { + return default_; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/TypeofTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/TypeofTypeExpr.java index 5f8d6b65cf7e..cf308191a030 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/TypeofTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/TypeofTypeExpr.java @@ -4,23 +4,23 @@ import com.semmle.js.ast.Visitor; /** - * A type of form typeof E where E is an expression that takes - * the form of a qualified name. + * A type of form typeof E where E is an expression that takes the form of a + * qualified name. */ public class TypeofTypeExpr extends TypeExpression { - private final ITypeExpression expression; // Always Identifier or MemberExpression. + private final ITypeExpression expression; // Always Identifier or MemberExpression. - public TypeofTypeExpr(SourceLocation loc, ITypeExpression expression) { - super("TypeofTypeExpr", loc); - this.expression = expression; - } + public TypeofTypeExpr(SourceLocation loc, ITypeExpression expression) { + super("TypeofTypeExpr", loc); + this.expression = expression; + } - public ITypeExpression getExpression() { - return expression; - } + public ITypeExpression getExpression() { + return expression; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/ast/UnionTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/UnionTypeExpr.java index 8de726e6ed57..4be288914f44 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/UnionTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/UnionTypeExpr.java @@ -1,26 +1,25 @@ package com.semmle.ts.ast; -import java.util.List; - import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; +import java.util.List; /** A union type such as number | string | boolean. */ public class UnionTypeExpr extends TypeExpression { - private final List elementTypes; + private final List elementTypes; - public UnionTypeExpr(SourceLocation loc, List elementTypes) { - super("UnionTypeExpr", loc); - this.elementTypes = elementTypes; - } + public UnionTypeExpr(SourceLocation loc, List elementTypes) { + super("UnionTypeExpr", loc); + this.elementTypes = elementTypes; + } - /** The members of the union; always contains at least two types. */ - public List getElementTypes() { - return elementTypes; - } + /** The members of the union; always contains at least two types. */ + public List getElementTypes() { + return elementTypes; + } - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } } diff --git a/javascript/extractor/src/com/semmle/ts/extractor/TypeExtractor.java b/javascript/extractor/src/com/semmle/ts/extractor/TypeExtractor.java index efe153bf3968..b626aab52d0b 100644 --- a/javascript/extractor/src/com/semmle/ts/extractor/TypeExtractor.java +++ b/javascript/extractor/src/com/semmle/ts/extractor/TypeExtractor.java @@ -1,251 +1,272 @@ package com.semmle.ts.extractor; -import java.util.LinkedHashMap; -import java.util.Map; - import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.semmle.util.trap.TrapWriter; import com.semmle.util.trap.TrapWriter.Label; +import java.util.LinkedHashMap; +import java.util.Map; /** * Extracts type and symbol information into TRAP files. - *

      - * This is closely coupled with the type_table.ts file in the - * parser-wrapper. Type strings and symbol strings generated in that file are - * parsed here. See that file for reference and documentation. + * + *

      This is closely coupled with the type_table.ts file in the parser-wrapper. Type + * strings and symbol strings generated in that file are parsed here. See that file for reference + * and documentation. */ public class TypeExtractor { - private final TrapWriter trapWriter; - private final TypeTable table; - - private static final Map tagToKind = new LinkedHashMap(); - - private static final int referenceKind = 6; - private static final int objectKind = 7; - private static final int typevarKind = 8; - private static final int typeofKind = 9; - private static final int uniqueSymbolKind = 15; - private static final int tupleKind = 18; - private static final int lexicalTypevarKind = 19; - private static final int thisKind = 20; - private static final int numberLiteralTypeKind = 21; - private static final int stringLiteralTypeKind = 22; - private static final int bigintLiteralTypeKind = 25; - - static { - tagToKind.put("any", 0); - tagToKind.put("string", 1); - tagToKind.put("number", 2); - tagToKind.put("union", 3); - tagToKind.put("true", 4); - tagToKind.put("false", 5); - tagToKind.put("reference", referenceKind); - tagToKind.put("object", objectKind); - tagToKind.put("typevar", typevarKind); - tagToKind.put("typeof", typeofKind); - tagToKind.put("void", 10); - tagToKind.put("undefined", 11); - tagToKind.put("null", 12); - tagToKind.put("never", 13); - tagToKind.put("plainsymbol", 14); - tagToKind.put("uniquesymbol", uniqueSymbolKind); - tagToKind.put("objectkeyword", 16); - tagToKind.put("intersection", 17); - tagToKind.put("tuple", tupleKind); - tagToKind.put("lextypevar", lexicalTypevarKind); - tagToKind.put("this", thisKind); - tagToKind.put("numlit", numberLiteralTypeKind); - tagToKind.put("strlit", stringLiteralTypeKind); - tagToKind.put("unknown", 23); - tagToKind.put("bigint", 24); - tagToKind.put("bigintlit", bigintLiteralTypeKind); - } - - private static final Map symbolKind = new LinkedHashMap(); - - static { - symbolKind.put("root", 0); - symbolKind.put("member", 1); - symbolKind.put("other", 2); - } - - public TypeExtractor(TrapWriter trapWriter, TypeTable table) { - this.trapWriter = trapWriter; - this.table = table; - } - - public void extract() { - for (int i = 0; i < table.getNumberOfTypes(); ++i) { - extractType(i); - } - extractPropertyLookups(table.getPropertyLookups()); - for (int i = 0; i < table.getNumberOfSymbols(); ++i) { - extractSymbol(i); - } - extractSymbolNameMapping("symbol_module", table.getModuleMappings()); - extractSymbolNameMapping("symbol_global", table.getGlobalMappings()); - extractSignatureMappings(table.getSignatureMappings()); - for (int i = 0; i < table.getNumberOfSignatures(); ++i) { - extractSignature(i); - } - extractIndexTypeTable(table.getNumberIndexTypes(), "number_index_type"); - extractIndexTypeTable(table.getStringIndexTypes(), "string_index_type"); - extractBaseTypes(table.getBaseTypes()); - extractSelfTypes(table.getSelfTypes()); - } - - private void extractType(int id) { - Label lbl = trapWriter.globalID("type;" + id); - String contents = table.getTypeString(id); - String[] parts = contents.split(";"); - int kind = tagToKind.get(parts[0]); - trapWriter.addTuple("types", lbl, kind, table.getTypeToStringValue(id)); - int firstChild = 1; - switch (kind) { - case referenceKind: - case typevarKind: - case typeofKind: - case uniqueSymbolKind: { - // The first part of a reference is the symbol for name binding. - Label symbol = trapWriter.globalID("symbol;" + parts[1]); - trapWriter.addTuple("type_symbol", lbl, symbol); - ++firstChild; - break; - } - case tupleKind: { - // The first two parts denote minimum length and presence of rest element. - trapWriter.addTuple("tuple_type_min_length", lbl, Integer.parseInt(parts[1])); - if (parts[2].equals("t")) { - trapWriter.addTuple("tuple_type_rest", lbl); - } - firstChild += 2; - break; - } - case objectKind: - case lexicalTypevarKind: - firstChild = parts.length; // No children. - break; - - case numberLiteralTypeKind: - case stringLiteralTypeKind: - case bigintLiteralTypeKind: - firstChild = parts.length; // No children. - // The string value may contain `;` so don't use the split(). - String value = contents.substring(parts[0].length() + 1); - trapWriter.addTuple("type_literal_value", lbl, value); - break; - } - for (int i = firstChild; i < parts.length; ++i) { - Label childLabel = trapWriter.globalID("type;" + parts[i]); - trapWriter.addTuple("type_child", childLabel, lbl, i - firstChild); - } - } - - private void extractPropertyLookups(JsonObject lookups) { - JsonArray baseTypes = lookups.get("baseTypes").getAsJsonArray(); - JsonArray names = lookups.get("names").getAsJsonArray(); - JsonArray propertyTypes = lookups.get("propertyTypes").getAsJsonArray(); - for (int i = 0; i < baseTypes.size(); ++i) { - int baseType = baseTypes.get(i).getAsInt(); - String name = names.get(i).getAsString(); - int propertyType = propertyTypes.get(i).getAsInt(); - trapWriter.addTuple("type_property", trapWriter.globalID("type;" + baseType), name, - trapWriter.globalID("type;" + propertyType)); - } - } - - private void extractSymbol(int index) { - // Format is: kind;decl;name[;parent] - String[] parts = table.getSymbolString(index).split(";"); - int kind = symbolKind.get(parts[0]); - String name = parts[2]; - Label label = trapWriter.globalID("symbol;" + index); - trapWriter.addTuple("symbols", label, kind, name); - if (parts.length == 4) { - Label parentLabel = trapWriter.globalID("symbol;" + parts[3]); - trapWriter.addTuple("symbol_parent", label, parentLabel); - } - } - - private void extractSymbolNameMapping(String relationName, JsonObject mappings) { - JsonArray symbols = mappings.get("symbols").getAsJsonArray(); - JsonArray names = mappings.get("names").getAsJsonArray(); - for (int i = 0; i < symbols.size(); ++i) { - Label symbol = trapWriter.globalID("symbol;" + symbols.get(i).getAsInt()); - String moduleName = names.get(i).getAsString(); - trapWriter.addTuple(relationName, symbol, moduleName); - } - } - - private void extractSignature(int index) { - // Format is: - // kind;numTypeParams;requiredParams;returnType(;paramName;paramType)* - String[] parts = table.getSignatureString(index).split(";"); - Label label = trapWriter.globalID("signature;" + index); - int kind = Integer.parseInt(parts[0]); - int numberOfTypeParameters = Integer.parseInt(parts[1]); - int requiredParameters = Integer.parseInt(parts[2]); - Label returnType = trapWriter.globalID("type;" + parts[3]); - trapWriter.addTuple("signature_types", label, kind, table.getSignatureToStringValue(index), numberOfTypeParameters, - requiredParameters); - trapWriter.addTuple("signature_contains_type", returnType, label, -1); - int numberOfParameters = (parts.length - 4) / 2; // includes type parameters - for (int i = 0; i < numberOfParameters; ++i) { - int partIndex = 4 + (2 * i); - String paramName = parts[partIndex]; - String paramTypeId = parts[partIndex + 1]; - if (paramTypeId.length() > 0) { // Unconstrained type parameters have an empty type ID. - Label paramType = trapWriter.globalID("type;" + parts[partIndex + 1]); - trapWriter.addTuple("signature_contains_type", paramType, label, i); - } - trapWriter.addTuple("signature_parameter_name", label, i, paramName); - } - } - - private void extractSignatureMappings(JsonObject mappings) { - JsonArray baseTypes = mappings.get("baseTypes").getAsJsonArray(); - JsonArray kinds = mappings.get("kinds").getAsJsonArray(); - JsonArray indices = mappings.get("indices").getAsJsonArray(); - JsonArray signatures = mappings.get("signatures").getAsJsonArray(); - for (int i = 0; i < baseTypes.size(); ++i) { - int baseType = baseTypes.get(i).getAsInt(); - int kind = kinds.get(i).getAsInt(); - int index = indices.get(i).getAsInt(); - int signatureId = signatures.get(i).getAsInt(); - trapWriter.addTuple("type_contains_signature", trapWriter.globalID("type;" + baseType), kind, index, - trapWriter.globalID("signature;" + signatureId)); - } - } - - private void extractIndexTypeTable(JsonObject table, String relationName) { - JsonArray baseTypes = table.get("baseTypes").getAsJsonArray(); - JsonArray propertyTypes = table.get("propertyTypes").getAsJsonArray(); - for (int i = 0; i < baseTypes.size(); ++i) { - int baseType = baseTypes.get(i).getAsInt(); - int propertyType = propertyTypes.get(i).getAsInt(); - trapWriter.addTuple(relationName, trapWriter.globalID("type;" + baseType), trapWriter.globalID("type;" + propertyType)); - } - } - - private void extractBaseTypes(JsonObject table) { - JsonArray symbols = table.get("symbols").getAsJsonArray(); - JsonArray baseTypeSymbols = table.get("baseTypeSymbols").getAsJsonArray(); - for (int i = 0; i < symbols.size(); ++i) { - int symbolId = symbols.get(i).getAsInt(); - int baseTypeSymbolId = baseTypeSymbols.get(i).getAsInt(); - trapWriter.addTuple("base_type_names", trapWriter.globalID("symbol;" + symbolId), - trapWriter.globalID("symbol;" + baseTypeSymbolId)); - } - } - - private void extractSelfTypes(JsonObject table) { - JsonArray symbols = table.get("symbols").getAsJsonArray(); - JsonArray selfTypes = table.get("selfTypes").getAsJsonArray(); - for (int i = 0; i < symbols.size(); ++i) { - int symbolId = symbols.get(i).getAsInt(); - int typeId = selfTypes.get(i).getAsInt(); - trapWriter.addTuple("self_types", trapWriter.globalID("symbol;" + symbolId), trapWriter.globalID("type;" + typeId)); - } - } + private final TrapWriter trapWriter; + private final TypeTable table; + + private static final Map tagToKind = new LinkedHashMap(); + + private static final int referenceKind = 6; + private static final int objectKind = 7; + private static final int typevarKind = 8; + private static final int typeofKind = 9; + private static final int uniqueSymbolKind = 15; + private static final int tupleKind = 18; + private static final int lexicalTypevarKind = 19; + private static final int thisKind = 20; + private static final int numberLiteralTypeKind = 21; + private static final int stringLiteralTypeKind = 22; + private static final int bigintLiteralTypeKind = 25; + + static { + tagToKind.put("any", 0); + tagToKind.put("string", 1); + tagToKind.put("number", 2); + tagToKind.put("union", 3); + tagToKind.put("true", 4); + tagToKind.put("false", 5); + tagToKind.put("reference", referenceKind); + tagToKind.put("object", objectKind); + tagToKind.put("typevar", typevarKind); + tagToKind.put("typeof", typeofKind); + tagToKind.put("void", 10); + tagToKind.put("undefined", 11); + tagToKind.put("null", 12); + tagToKind.put("never", 13); + tagToKind.put("plainsymbol", 14); + tagToKind.put("uniquesymbol", uniqueSymbolKind); + tagToKind.put("objectkeyword", 16); + tagToKind.put("intersection", 17); + tagToKind.put("tuple", tupleKind); + tagToKind.put("lextypevar", lexicalTypevarKind); + tagToKind.put("this", thisKind); + tagToKind.put("numlit", numberLiteralTypeKind); + tagToKind.put("strlit", stringLiteralTypeKind); + tagToKind.put("unknown", 23); + tagToKind.put("bigint", 24); + tagToKind.put("bigintlit", bigintLiteralTypeKind); + } + + private static final Map symbolKind = new LinkedHashMap(); + + static { + symbolKind.put("root", 0); + symbolKind.put("member", 1); + symbolKind.put("other", 2); + } + + public TypeExtractor(TrapWriter trapWriter, TypeTable table) { + this.trapWriter = trapWriter; + this.table = table; + } + + public void extract() { + for (int i = 0; i < table.getNumberOfTypes(); ++i) { + extractType(i); + } + extractPropertyLookups(table.getPropertyLookups()); + for (int i = 0; i < table.getNumberOfSymbols(); ++i) { + extractSymbol(i); + } + extractSymbolNameMapping("symbol_module", table.getModuleMappings()); + extractSymbolNameMapping("symbol_global", table.getGlobalMappings()); + extractSignatureMappings(table.getSignatureMappings()); + for (int i = 0; i < table.getNumberOfSignatures(); ++i) { + extractSignature(i); + } + extractIndexTypeTable(table.getNumberIndexTypes(), "number_index_type"); + extractIndexTypeTable(table.getStringIndexTypes(), "string_index_type"); + extractBaseTypes(table.getBaseTypes()); + extractSelfTypes(table.getSelfTypes()); + } + + private void extractType(int id) { + Label lbl = trapWriter.globalID("type;" + id); + String contents = table.getTypeString(id); + String[] parts = contents.split(";"); + int kind = tagToKind.get(parts[0]); + trapWriter.addTuple("types", lbl, kind, table.getTypeToStringValue(id)); + int firstChild = 1; + switch (kind) { + case referenceKind: + case typevarKind: + case typeofKind: + case uniqueSymbolKind: + { + // The first part of a reference is the symbol for name binding. + Label symbol = trapWriter.globalID("symbol;" + parts[1]); + trapWriter.addTuple("type_symbol", lbl, symbol); + ++firstChild; + break; + } + case tupleKind: + { + // The first two parts denote minimum length and presence of rest element. + trapWriter.addTuple("tuple_type_min_length", lbl, Integer.parseInt(parts[1])); + if (parts[2].equals("t")) { + trapWriter.addTuple("tuple_type_rest", lbl); + } + firstChild += 2; + break; + } + case objectKind: + case lexicalTypevarKind: + firstChild = parts.length; // No children. + break; + + case numberLiteralTypeKind: + case stringLiteralTypeKind: + case bigintLiteralTypeKind: + firstChild = parts.length; // No children. + // The string value may contain `;` so don't use the split(). + String value = contents.substring(parts[0].length() + 1); + trapWriter.addTuple("type_literal_value", lbl, value); + break; + } + for (int i = firstChild; i < parts.length; ++i) { + Label childLabel = trapWriter.globalID("type;" + parts[i]); + trapWriter.addTuple("type_child", childLabel, lbl, i - firstChild); + } + } + + private void extractPropertyLookups(JsonObject lookups) { + JsonArray baseTypes = lookups.get("baseTypes").getAsJsonArray(); + JsonArray names = lookups.get("names").getAsJsonArray(); + JsonArray propertyTypes = lookups.get("propertyTypes").getAsJsonArray(); + for (int i = 0; i < baseTypes.size(); ++i) { + int baseType = baseTypes.get(i).getAsInt(); + String name = names.get(i).getAsString(); + int propertyType = propertyTypes.get(i).getAsInt(); + trapWriter.addTuple( + "type_property", + trapWriter.globalID("type;" + baseType), + name, + trapWriter.globalID("type;" + propertyType)); + } + } + + private void extractSymbol(int index) { + // Format is: kind;decl;name[;parent] + String[] parts = table.getSymbolString(index).split(";"); + int kind = symbolKind.get(parts[0]); + String name = parts[2]; + Label label = trapWriter.globalID("symbol;" + index); + trapWriter.addTuple("symbols", label, kind, name); + if (parts.length == 4) { + Label parentLabel = trapWriter.globalID("symbol;" + parts[3]); + trapWriter.addTuple("symbol_parent", label, parentLabel); + } + } + + private void extractSymbolNameMapping(String relationName, JsonObject mappings) { + JsonArray symbols = mappings.get("symbols").getAsJsonArray(); + JsonArray names = mappings.get("names").getAsJsonArray(); + for (int i = 0; i < symbols.size(); ++i) { + Label symbol = trapWriter.globalID("symbol;" + symbols.get(i).getAsInt()); + String moduleName = names.get(i).getAsString(); + trapWriter.addTuple(relationName, symbol, moduleName); + } + } + + private void extractSignature(int index) { + // Format is: + // kind;numTypeParams;requiredParams;returnType(;paramName;paramType)* + String[] parts = table.getSignatureString(index).split(";"); + Label label = trapWriter.globalID("signature;" + index); + int kind = Integer.parseInt(parts[0]); + int numberOfTypeParameters = Integer.parseInt(parts[1]); + int requiredParameters = Integer.parseInt(parts[2]); + Label returnType = trapWriter.globalID("type;" + parts[3]); + trapWriter.addTuple( + "signature_types", + label, + kind, + table.getSignatureToStringValue(index), + numberOfTypeParameters, + requiredParameters); + trapWriter.addTuple("signature_contains_type", returnType, label, -1); + int numberOfParameters = (parts.length - 4) / 2; // includes type parameters + for (int i = 0; i < numberOfParameters; ++i) { + int partIndex = 4 + (2 * i); + String paramName = parts[partIndex]; + String paramTypeId = parts[partIndex + 1]; + if (paramTypeId.length() > 0) { // Unconstrained type parameters have an empty type ID. + Label paramType = trapWriter.globalID("type;" + parts[partIndex + 1]); + trapWriter.addTuple("signature_contains_type", paramType, label, i); + } + trapWriter.addTuple("signature_parameter_name", label, i, paramName); + } + } + + private void extractSignatureMappings(JsonObject mappings) { + JsonArray baseTypes = mappings.get("baseTypes").getAsJsonArray(); + JsonArray kinds = mappings.get("kinds").getAsJsonArray(); + JsonArray indices = mappings.get("indices").getAsJsonArray(); + JsonArray signatures = mappings.get("signatures").getAsJsonArray(); + for (int i = 0; i < baseTypes.size(); ++i) { + int baseType = baseTypes.get(i).getAsInt(); + int kind = kinds.get(i).getAsInt(); + int index = indices.get(i).getAsInt(); + int signatureId = signatures.get(i).getAsInt(); + trapWriter.addTuple( + "type_contains_signature", + trapWriter.globalID("type;" + baseType), + kind, + index, + trapWriter.globalID("signature;" + signatureId)); + } + } + + private void extractIndexTypeTable(JsonObject table, String relationName) { + JsonArray baseTypes = table.get("baseTypes").getAsJsonArray(); + JsonArray propertyTypes = table.get("propertyTypes").getAsJsonArray(); + for (int i = 0; i < baseTypes.size(); ++i) { + int baseType = baseTypes.get(i).getAsInt(); + int propertyType = propertyTypes.get(i).getAsInt(); + trapWriter.addTuple( + relationName, + trapWriter.globalID("type;" + baseType), + trapWriter.globalID("type;" + propertyType)); + } + } + + private void extractBaseTypes(JsonObject table) { + JsonArray symbols = table.get("symbols").getAsJsonArray(); + JsonArray baseTypeSymbols = table.get("baseTypeSymbols").getAsJsonArray(); + for (int i = 0; i < symbols.size(); ++i) { + int symbolId = symbols.get(i).getAsInt(); + int baseTypeSymbolId = baseTypeSymbols.get(i).getAsInt(); + trapWriter.addTuple( + "base_type_names", + trapWriter.globalID("symbol;" + symbolId), + trapWriter.globalID("symbol;" + baseTypeSymbolId)); + } + } + + private void extractSelfTypes(JsonObject table) { + JsonArray symbols = table.get("symbols").getAsJsonArray(); + JsonArray selfTypes = table.get("selfTypes").getAsJsonArray(); + for (int i = 0; i < symbols.size(); ++i) { + int symbolId = symbols.get(i).getAsInt(); + int typeId = selfTypes.get(i).getAsInt(); + trapWriter.addTuple( + "self_types", + trapWriter.globalID("symbol;" + symbolId), + trapWriter.globalID("type;" + typeId)); + } + } } diff --git a/javascript/extractor/src/com/semmle/ts/extractor/TypeTable.java b/javascript/extractor/src/com/semmle/ts/extractor/TypeTable.java index c2bd5156cb18..886ee6184243 100644 --- a/javascript/extractor/src/com/semmle/ts/extractor/TypeTable.java +++ b/javascript/extractor/src/com/semmle/ts/extractor/TypeTable.java @@ -5,109 +5,109 @@ /** * Holds the output of the get-type-table command. - *

      - * See documentation in parser-wrapper/src/type_table.ts. + * + *

      See documentation in parser-wrapper/src/type_table.ts. */ public class TypeTable { - private final JsonArray typeStrings; - private final JsonArray typeToStringValues; - private final JsonObject propertyLookups; - private final JsonArray symbolStrings; - private final JsonObject moduleMappings; - private final JsonObject globalMappings; - private final JsonArray signatureStrings; - private final JsonObject signatureMappings; - private final JsonArray signatureToStringValues; - private final JsonObject stringIndexTypes; - private final JsonObject numberIndexTypes; - private final JsonObject baseTypes; - private final JsonObject selfTypes; - - public TypeTable(JsonObject typeTable) { - this.typeStrings = typeTable.get("typeStrings").getAsJsonArray(); - this.typeToStringValues = typeTable.get("typeToStringValues").getAsJsonArray(); - this.propertyLookups = typeTable.get("propertyLookups").getAsJsonObject(); - this.symbolStrings = typeTable.get("symbolStrings").getAsJsonArray(); - this.moduleMappings = typeTable.get("moduleMappings").getAsJsonObject(); - this.globalMappings = typeTable.get("globalMappings").getAsJsonObject(); - this.signatureStrings = typeTable.get("signatureStrings").getAsJsonArray(); - this.signatureMappings = typeTable.get("signatureMappings").getAsJsonObject(); - this.signatureToStringValues = typeTable.get("signatureToStringValues").getAsJsonArray(); - this.numberIndexTypes = typeTable.get("numberIndexTypes").getAsJsonObject(); - this.stringIndexTypes = typeTable.get("stringIndexTypes").getAsJsonObject(); - this.baseTypes = typeTable.get("baseTypes").getAsJsonObject(); - this.selfTypes = typeTable.get("selfTypes").getAsJsonObject(); - } - - public String getTypeString(int index) { - return typeStrings.get(index).getAsString(); - } - - public String getTypeToStringValue(int index) { - return typeToStringValues.get(index).getAsString(); - } - - public JsonObject getPropertyLookups() { - return propertyLookups; - } - - public int getNumberOfTypes() { - return typeStrings.size(); - } - - public String getSymbolString(int index) { - return symbolStrings.get(index).getAsString(); - } - - public int getNumberOfSymbols() { - return symbolStrings.size(); - } - - public JsonObject getModuleMappings() { - return moduleMappings; - } - - public JsonObject getGlobalMappings() { - return globalMappings; - } - - public JsonArray getSignatureStrings() { - return signatureStrings; - } - - public int getNumberOfSignatures() { - return signatureStrings.size(); - } - - public String getSignatureString(int i) { - return signatureStrings.get(i).getAsString(); - } - - public JsonObject getSignatureMappings() { - return signatureMappings; - } - - public JsonArray getSignatureToStringValues() { - return signatureToStringValues; - } - - public String getSignatureToStringValue(int i) { - return signatureToStringValues.get(i).getAsString(); - } - - public JsonObject getNumberIndexTypes() { - return numberIndexTypes; - } - - public JsonObject getStringIndexTypes() { - return stringIndexTypes; - } - - public JsonObject getBaseTypes() { - return baseTypes; - } - - public JsonObject getSelfTypes() { - return selfTypes; - } + private final JsonArray typeStrings; + private final JsonArray typeToStringValues; + private final JsonObject propertyLookups; + private final JsonArray symbolStrings; + private final JsonObject moduleMappings; + private final JsonObject globalMappings; + private final JsonArray signatureStrings; + private final JsonObject signatureMappings; + private final JsonArray signatureToStringValues; + private final JsonObject stringIndexTypes; + private final JsonObject numberIndexTypes; + private final JsonObject baseTypes; + private final JsonObject selfTypes; + + public TypeTable(JsonObject typeTable) { + this.typeStrings = typeTable.get("typeStrings").getAsJsonArray(); + this.typeToStringValues = typeTable.get("typeToStringValues").getAsJsonArray(); + this.propertyLookups = typeTable.get("propertyLookups").getAsJsonObject(); + this.symbolStrings = typeTable.get("symbolStrings").getAsJsonArray(); + this.moduleMappings = typeTable.get("moduleMappings").getAsJsonObject(); + this.globalMappings = typeTable.get("globalMappings").getAsJsonObject(); + this.signatureStrings = typeTable.get("signatureStrings").getAsJsonArray(); + this.signatureMappings = typeTable.get("signatureMappings").getAsJsonObject(); + this.signatureToStringValues = typeTable.get("signatureToStringValues").getAsJsonArray(); + this.numberIndexTypes = typeTable.get("numberIndexTypes").getAsJsonObject(); + this.stringIndexTypes = typeTable.get("stringIndexTypes").getAsJsonObject(); + this.baseTypes = typeTable.get("baseTypes").getAsJsonObject(); + this.selfTypes = typeTable.get("selfTypes").getAsJsonObject(); + } + + public String getTypeString(int index) { + return typeStrings.get(index).getAsString(); + } + + public String getTypeToStringValue(int index) { + return typeToStringValues.get(index).getAsString(); + } + + public JsonObject getPropertyLookups() { + return propertyLookups; + } + + public int getNumberOfTypes() { + return typeStrings.size(); + } + + public String getSymbolString(int index) { + return symbolStrings.get(index).getAsString(); + } + + public int getNumberOfSymbols() { + return symbolStrings.size(); + } + + public JsonObject getModuleMappings() { + return moduleMappings; + } + + public JsonObject getGlobalMappings() { + return globalMappings; + } + + public JsonArray getSignatureStrings() { + return signatureStrings; + } + + public int getNumberOfSignatures() { + return signatureStrings.size(); + } + + public String getSignatureString(int i) { + return signatureStrings.get(i).getAsString(); + } + + public JsonObject getSignatureMappings() { + return signatureMappings; + } + + public JsonArray getSignatureToStringValues() { + return signatureToStringValues; + } + + public String getSignatureToStringValue(int i) { + return signatureToStringValues.get(i).getAsString(); + } + + public JsonObject getNumberIndexTypes() { + return numberIndexTypes; + } + + public JsonObject getStringIndexTypes() { + return stringIndexTypes; + } + + public JsonObject getBaseTypes() { + return baseTypes; + } + + public JsonObject getSelfTypes() { + return selfTypes; + } } From e933ba28d5ed94428eea450396a9e85c01a8a9a1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 26 Feb 2019 16:26:47 +0000 Subject: [PATCH 44/71] Python: Add basic support for stdlib cookie objects. --- python/ql/src/semmle/python/web/Http.qll | 32 ++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/python/ql/src/semmle/python/web/Http.qll b/python/ql/src/semmle/python/web/Http.qll index 0f4d82b3c41a..a6a0a5e5ba48 100644 --- a/python/ql/src/semmle/python/web/Http.qll +++ b/python/ql/src/semmle/python/web/Http.qll @@ -56,4 +56,36 @@ class WsgiEnvironment extends TaintKind { } +/** A standard morsel object from a HTTP request, a value in a cookie, + * typically an instance of `http.cookies.Morsel` */ +class UntrustedMorsel extends TaintKind { + + UntrustedMorsel() { + this = "http.Morsel" + } + + + override TaintKind getTaintOfAttribute(string name) { + result instanceof ExternalStringKind and + ( + name = "value" + ) + } + +} + +/** A standard cookie object from a HTTP request, typically an instance of `http.cookies.SimpleCookie` */ +class UntrustedCookie extends TaintKind { + + UntrustedCookie() { + this = "http.Cookie" + } + + override TaintKind getTaintForFlowStep(ControlFlowNode fromnode, ControlFlowNode tonode) { + tonode.(SubscriptNode).getValue() = fromnode and + result instanceof UntrustedMorsel + } + +} + From 6c82be8bda5182933c6a3cb080f58370c564b9db Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 27 Feb 2019 15:14:42 +0000 Subject: [PATCH 45/71] Python: CherryPy web framework support -- requests. --- .../ql/src/semmle/python/web/HttpRequest.qll | 1 + .../semmle/python/web/cherrypy/General.qll | 54 +++++++++++++++ .../semmle/python/web/cherrypy/Request.qll | 69 +++++++++++++++++++ .../web/cherrypy/Sources.expected | 4 ++ .../library-tests/web/cherrypy/Sources.ql | 10 +++ .../test/library-tests/web/cherrypy/options | 2 + .../ql/test/library-tests/web/cherrypy/red.py | 11 +++ .../test/library-tests/web/cherrypy/test.py | 23 +++++++ .../Security/lib/cherrypy/__init__.py | 15 ++++ .../Security/lib/cherrypy/_helper.py | 31 +++++++++ 10 files changed, 220 insertions(+) create mode 100644 python/ql/src/semmle/python/web/cherrypy/General.qll create mode 100644 python/ql/src/semmle/python/web/cherrypy/Request.qll create mode 100644 python/ql/test/library-tests/web/cherrypy/Sources.expected create mode 100644 python/ql/test/library-tests/web/cherrypy/Sources.ql create mode 100644 python/ql/test/library-tests/web/cherrypy/options create mode 100644 python/ql/test/library-tests/web/cherrypy/red.py create mode 100644 python/ql/test/library-tests/web/cherrypy/test.py create mode 100644 python/ql/test/query-tests/Security/lib/cherrypy/__init__.py create mode 100644 python/ql/test/query-tests/Security/lib/cherrypy/_helper.py diff --git a/python/ql/src/semmle/python/web/HttpRequest.qll b/python/ql/src/semmle/python/web/HttpRequest.qll index 066f6c1e27fd..8c5ae5963ec0 100644 --- a/python/ql/src/semmle/python/web/HttpRequest.qll +++ b/python/ql/src/semmle/python/web/HttpRequest.qll @@ -6,3 +6,4 @@ import semmle.python.web.twisted.Request import semmle.python.web.bottle.Request import semmle.python.web.turbogears.Request import semmle.python.web.falcon.Request +import semmle.python.web.cherrypy.Request diff --git a/python/ql/src/semmle/python/web/cherrypy/General.qll b/python/ql/src/semmle/python/web/cherrypy/General.qll new file mode 100644 index 000000000000..6f950e3ab8b7 --- /dev/null +++ b/python/ql/src/semmle/python/web/cherrypy/General.qll @@ -0,0 +1,54 @@ +import python +import semmle.python.web.Http + +module CherryPy { + + FunctionObject expose() { + result = ModuleObject::named("cherrypy").attr("expose") + } + +} + +class CherryPyExposedFunction extends Function { + + CherryPyExposedFunction() { + this.getADecorator().refersTo(CherryPy::expose()) + } + +} + +class CherryPyRoute extends CallNode { + + CherryPyRoute() { + /* cherrypy.quickstart(root, script_name, config) */ + ModuleObject::named("cherrypy").attr("quickstart").(FunctionObject).getACall() = this + or + /* cherrypy.tree.mount(root, script_name, config) */ + this.getFunction().(AttrNode).getObject("mount").refersTo(ModuleObject::named("cherrypy").attr("tree")) + } + + ClassObject getAppClass() { + this.getArg(0).refersTo(_, result, _) + or + this.getArgByName("root").refersTo(_, result, _) + } + + string getPath() { + exists(StringObject path | + result = path.getText() + | + this.getArg(1).refersTo(path) + or + this.getArgByName("script_name").refersTo(path) + ) + } + + Object getConfig() { + this.getArg(2).refersTo(_, result, _) + or + this.getArgByName("config").refersTo(_, result, _) + } + +} + + diff --git a/python/ql/src/semmle/python/web/cherrypy/Request.qll b/python/ql/src/semmle/python/web/cherrypy/Request.qll new file mode 100644 index 000000000000..34450c9283ec --- /dev/null +++ b/python/ql/src/semmle/python/web/cherrypy/Request.qll @@ -0,0 +1,69 @@ +import python + +import semmle.python.security.TaintTracking +import semmle.python.security.strings.Basic +import semmle.python.web.Http +import semmle.python.web.cherrypy.General + +/** A twisted.web.http.Request object */ +class CherryPyRequest extends TaintKind { + + CherryPyRequest() { + this = "cherrypy.request" + } + + override TaintKind getTaintOfAttribute(string name) { + name = "params" and result instanceof ExternalStringDictKind + or + name = "cookie" and result instanceof UntrustedCookie + } + + override TaintKind getTaintOfMethodResult(string name) { + ( + name = "getHeader" or + name = "getCookie" or + name = "getUser" or + name = "getPassword" + ) and + result instanceof ExternalStringKind + } + +} + + +class CherryPyExposedFunctionParameter extends TaintSource { + + CherryPyExposedFunctionParameter() { + exists(Parameter p | + p = any(CherryPyExposedFunction f).getAnArg() and + not p.isSelf() and + p.asName().getAFlowNode() = this + ) + } + + override string toString() { + result = "CherryPy handler function parameter" + } + + override predicate isSourceOf(TaintKind kind) { + kind instanceof ExternalStringKind + } + +} + +class CherryPyRequestSource extends TaintSource { + + CherryPyRequestSource() { + this.(ControlFlowNode).refersTo(ModuleObject::named("cherrypy").attr("request")) + } + + override predicate isSourceOf(TaintKind kind) { + kind instanceof CherryPyRequest + } + +} + + + + + diff --git a/python/ql/test/library-tests/web/cherrypy/Sources.expected b/python/ql/test/library-tests/web/cherrypy/Sources.expected new file mode 100644 index 000000000000..73fdf0f2d68c --- /dev/null +++ b/python/ql/test/library-tests/web/cherrypy/Sources.expected @@ -0,0 +1,4 @@ +| ../../../query-tests/Security/lib/cherrypy/__init__.py:10 | _ThreadLocalProxy() | cherrypy.request | +| ../../../query-tests/Security/lib/cherrypy/__init__.py:10 | request | cherrypy.request | +| test.py:10 | arg | externally controlled string | +| test.py:16 | arg | externally controlled string | diff --git a/python/ql/test/library-tests/web/cherrypy/Sources.ql b/python/ql/test/library-tests/web/cherrypy/Sources.ql new file mode 100644 index 000000000000..c1b9cc82e197 --- /dev/null +++ b/python/ql/test/library-tests/web/cherrypy/Sources.ql @@ -0,0 +1,10 @@ + +import python + +import semmle.python.web.HttpRequest +import semmle.python.security.strings.Untrusted + + +from TaintSource src, TaintKind kind +where src.isSourceOf(kind) and not kind.matches("tornado%") +select src.getLocation().toString(), src.(ControlFlowNode).getNode().toString(), kind diff --git a/python/ql/test/library-tests/web/cherrypy/options b/python/ql/test/library-tests/web/cherrypy/options new file mode 100644 index 000000000000..3eb8fa37213e --- /dev/null +++ b/python/ql/test/library-tests/web/cherrypy/options @@ -0,0 +1,2 @@ +semmle-extractor-options: --max-import-depth=3 --lang=3 -p ../../../query-tests/Security/lib/ +optimize: true diff --git a/python/ql/test/library-tests/web/cherrypy/red.py b/python/ql/test/library-tests/web/cherrypy/red.py new file mode 100644 index 000000000000..5fa25b5aa0f5 --- /dev/null +++ b/python/ql/test/library-tests/web/cherrypy/red.py @@ -0,0 +1,11 @@ + +import cherrypy + +class MultiPath(object): + + @cherrypy.expose(['color', 'colour']) + def red(self): + return "RED" + +if __name__ == '__main__': + cherrypy.quickstart(MultiPath()) diff --git a/python/ql/test/library-tests/web/cherrypy/test.py b/python/ql/test/library-tests/web/cherrypy/test.py new file mode 100644 index 000000000000..5d44b54077bd --- /dev/null +++ b/python/ql/test/library-tests/web/cherrypy/test.py @@ -0,0 +1,23 @@ + +import random +import string + +import cherrypy + +class A(object): + + @cherrypy.expose + def a(self, arg): + return "hello " + arg + +class B(object): + + @cherrypy.expose + def b(self, arg): + return "bye " + arg + +cherrypy.tree.mount(A(), '/a', a_conf) +cherrypy.tree.mount(B(), '/b', b_conf) + +cherrypy.engine.start() +cherrypy.engine.block() \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/cherrypy/__init__.py b/python/ql/test/query-tests/Security/lib/cherrypy/__init__.py new file mode 100644 index 000000000000..a1ae780e3b5d --- /dev/null +++ b/python/ql/test/query-tests/Security/lib/cherrypy/__init__.py @@ -0,0 +1,15 @@ + + +from ._helper import expose, popargs, url + +class _ThreadLocalProxy(object): + def __getattr__(self, name): + pass + + +request = _ThreadLocalProxy('request') +response = _ThreadLocalProxy('response') + +def quickstart(root=None, script_name='', config=None): + """Mount the given root, start the builtin server (and engine), then block.""" + pass diff --git a/python/ql/test/query-tests/Security/lib/cherrypy/_helper.py b/python/ql/test/query-tests/Security/lib/cherrypy/_helper.py new file mode 100644 index 000000000000..fc3f3604a57b --- /dev/null +++ b/python/ql/test/query-tests/Security/lib/cherrypy/_helper.py @@ -0,0 +1,31 @@ +def expose(func=None, alias=None): + """Expose the function or class. + Optionally provide an alias or set of aliases. + """ + def expose_(func): + func.exposed = True + return func + + return expose_ + + +def popargs(*args, **kwargs): + """Decorate _cp_dispatch.""" + + def decorated(cls_or_self=None, vpath=None): + if inspect.isclass(cls_or_self): + # cherrypy.popargs is a class decorator + return cls + + # We're in the actual function + self = cls_or_self + if vpath: + return getattr(self, vpath.pop(0), None) + else: + return self + + return decorated + +def url(path='', qs='', script_name=None, base=None, relative=None): + #Do some opaque stuff here... + return new_url From 91a1cc9f0b3df0d2c0b2e8b9e46086a2b96b3169 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 27 Feb 2019 16:32:45 +0000 Subject: [PATCH 46/71] Python: Add cherrypy handler function return values as taint sinks. --- .../ql/src/semmle/python/web/HttpResponse.qll | 1 + .../semmle/python/web/cherrypy/General.qll | 2 ++ .../semmle/python/web/cherrypy/Response.qll | 28 +++++++++++++++++++ .../library-tests/web/cherrypy/Sinks.expected | 3 ++ .../test/library-tests/web/cherrypy/Sinks.ql | 10 +++++++ 5 files changed, 44 insertions(+) create mode 100644 python/ql/src/semmle/python/web/cherrypy/Response.qll create mode 100644 python/ql/test/library-tests/web/cherrypy/Sinks.expected create mode 100644 python/ql/test/library-tests/web/cherrypy/Sinks.ql diff --git a/python/ql/src/semmle/python/web/HttpResponse.qll b/python/ql/src/semmle/python/web/HttpResponse.qll index 3c5d45701a79..5354a7359986 100644 --- a/python/ql/src/semmle/python/web/HttpResponse.qll +++ b/python/ql/src/semmle/python/web/HttpResponse.qll @@ -6,3 +6,4 @@ import semmle.python.web.twisted.Response import semmle.python.web.bottle.Response import semmle.python.web.turbogears.Response import semmle.python.web.falcon.Response +import semmle.python.web.cherrypy.Response diff --git a/python/ql/src/semmle/python/web/cherrypy/General.qll b/python/ql/src/semmle/python/web/cherrypy/General.qll index 6f950e3ab8b7..4b7e3ecf8eab 100644 --- a/python/ql/src/semmle/python/web/cherrypy/General.qll +++ b/python/ql/src/semmle/python/web/cherrypy/General.qll @@ -13,6 +13,8 @@ class CherryPyExposedFunction extends Function { CherryPyExposedFunction() { this.getADecorator().refersTo(CherryPy::expose()) + or + this.getADecorator().(Call).getFunc().refersTo(CherryPy::expose()) } } diff --git a/python/ql/src/semmle/python/web/cherrypy/Response.qll b/python/ql/src/semmle/python/web/cherrypy/Response.qll new file mode 100644 index 000000000000..773cd3575416 --- /dev/null +++ b/python/ql/src/semmle/python/web/cherrypy/Response.qll @@ -0,0 +1,28 @@ +import python + +import semmle.python.security.TaintTracking +import semmle.python.security.strings.Untrusted +import semmle.python.web.Http +import semmle.python.web.cherrypy.General + + + +class CherryPyExposedFunctionResult extends TaintSink { + + CherryPyExposedFunctionResult() { + exists(Return ret | + ret.getScope() instanceof CherryPyExposedFunction and + ret.getValue().getAFlowNode() = this + ) + } + + override predicate sinks(TaintKind kind) { + kind instanceof StringKind + } + + override string toString() { + result = "cherrypy handler function result" + } + +} + diff --git a/python/ql/test/library-tests/web/cherrypy/Sinks.expected b/python/ql/test/library-tests/web/cherrypy/Sinks.expected new file mode 100644 index 000000000000..e47936b055c9 --- /dev/null +++ b/python/ql/test/library-tests/web/cherrypy/Sinks.expected @@ -0,0 +1,3 @@ +| red.py:8 | Str | externally controlled string | +| test.py:11 | BinaryExpr | externally controlled string | +| test.py:17 | BinaryExpr | externally controlled string | diff --git a/python/ql/test/library-tests/web/cherrypy/Sinks.ql b/python/ql/test/library-tests/web/cherrypy/Sinks.ql new file mode 100644 index 000000000000..34aa1cfc429c --- /dev/null +++ b/python/ql/test/library-tests/web/cherrypy/Sinks.ql @@ -0,0 +1,10 @@ + +import python + +import semmle.python.web.HttpRequest +import semmle.python.web.HttpResponse +import semmle.python.security.strings.Untrusted + +from TaintSink sink, TaintKind kind +where sink.sinks(kind) +select sink.getLocation().toString(), sink.(ControlFlowNode).getNode().toString(), kind From 2df718d63244c9573abbbc0561cf1710ec98dd27 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 27 Feb 2019 16:45:31 +0000 Subject: [PATCH 47/71] Python: Make bottle response logic consistent with other frameworks. --- python/ql/src/semmle/python/web/bottle/Response.qll | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ql/src/semmle/python/web/bottle/Response.qll b/python/ql/src/semmle/python/web/bottle/Response.qll index 08ff2d1ffaae..c5fb81c2bab8 100644 --- a/python/ql/src/semmle/python/web/bottle/Response.qll +++ b/python/ql/src/semmle/python/web/bottle/Response.qll @@ -32,7 +32,7 @@ class BottleResponseBodyAssignment extends TaintSink { } override predicate sinks(TaintKind kind) { - kind instanceof UntrustedStringKind + kind instanceof StringKind } } @@ -47,7 +47,7 @@ class BottleHandlerFunctionResult extends TaintSink { } override predicate sinks(TaintKind kind) { - kind instanceof UntrustedStringKind + kind instanceof StringKind } override string toString() { From faf9b4886d23ae9df726a466f02dfc78abdc0ec4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 27 Feb 2019 16:51:23 +0000 Subject: [PATCH 48/71] Python: Add change note for CherryPy support. --- change-notes/1.20/analysis-python.md | 1 + change-notes/1.20/support/python-frameworks.csv | 1 + 2 files changed, 2 insertions(+) diff --git a/change-notes/1.20/analysis-python.md b/change-notes/1.20/analysis-python.md index 2b6b2ff47a02..c28bfbb0edc0 100644 --- a/change-notes/1.20/analysis-python.md +++ b/change-notes/1.20/analysis-python.md @@ -40,6 +40,7 @@ The API has been improved to declutter the global namespace and improve discover * Added support for the `dill` pickle library. * Added support for the `bottle` web framework. + * Added support for the `CherryPy` web framework. * Added support for the `falcon` web API framework. * Added support for the `turbogears` web framework. diff --git a/change-notes/1.20/support/python-frameworks.csv b/change-notes/1.20/support/python-frameworks.csv index 08a10d41ef9b..7a5e78d87568 100644 --- a/change-notes/1.20/support/python-frameworks.csv +++ b/change-notes/1.20/support/python-frameworks.csv @@ -1,5 +1,6 @@ Name, Category Bottle, Web framework +CherryPy, Web framework Django, Web application framework Falcon, Web API framework Flask, Microframework From af2680729f3dd546e3cf9abb4b34d72302f76f4e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 28 Feb 2019 11:29:03 +0000 Subject: [PATCH 49/71] Python: Fix qldoc. --- python/ql/src/semmle/python/web/cherrypy/Request.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ql/src/semmle/python/web/cherrypy/Request.qll b/python/ql/src/semmle/python/web/cherrypy/Request.qll index 34450c9283ec..71976fa0a264 100644 --- a/python/ql/src/semmle/python/web/cherrypy/Request.qll +++ b/python/ql/src/semmle/python/web/cherrypy/Request.qll @@ -5,7 +5,7 @@ import semmle.python.security.strings.Basic import semmle.python.web.Http import semmle.python.web.cherrypy.General -/** A twisted.web.http.Request object */ +/** The cherrypy.request local-proxy object */ class CherryPyRequest extends TaintKind { CherryPyRequest() { From a709a2d0f3e8941c82403829d0bb3033da639964 Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Wed, 27 Feb 2019 13:58:34 +0000 Subject: [PATCH 50/71] C++: Add Variable.isConstexpr() --- change-notes/1.20/analysis-cpp.md | 1 + cpp/ql/src/semmle/code/cpp/Variable.qll | 7 +++++++ .../library-tests/variables/constexpr/constexpr.cpp | 6 ++++++ .../variables/constexpr/constexpr.expected | 10 ++++++++++ .../library-tests/variables/constexpr/constexpr.ql | 5 +++++ 5 files changed, 29 insertions(+) create mode 100644 cpp/ql/test/library-tests/variables/constexpr/constexpr.cpp create mode 100644 cpp/ql/test/library-tests/variables/constexpr/constexpr.expected create mode 100644 cpp/ql/test/library-tests/variables/constexpr/constexpr.ql diff --git a/change-notes/1.20/analysis-cpp.md b/change-notes/1.20/analysis-cpp.md index 578fbbc99660..46a0929850e6 100644 --- a/change-notes/1.20/analysis-cpp.md +++ b/change-notes/1.20/analysis-cpp.md @@ -37,3 +37,4 @@ * There is a new `Namespace.isInline()` predicate, which holds if the namespace was declared as `inline namespace`. * The `Expr.isConstant()` predicate now also holds for _address constant expressions_, which are addresses that will be constant after the program has been linked. These address constants do not have a result for `Expr.getValue()`. * There are new `Function.isDeclaredConstexpr()` and `Function.isConstexpr()` predicates. They can be used to tell whether a function was declared as `constexpr`, and whether it actually is `constexpr`. +* There is a new `Variable.isConstexpr()` predicates. It can be used to tell whether a variable is `constexpr`. diff --git a/cpp/ql/src/semmle/code/cpp/Variable.qll b/cpp/ql/src/semmle/code/cpp/Variable.qll index c48f52a63a40..c535f742596f 100644 --- a/cpp/ql/src/semmle/code/cpp/Variable.qll +++ b/cpp/ql/src/semmle/code/cpp/Variable.qll @@ -121,6 +121,13 @@ class Variable extends Declaration, @variable { result.getLValue() = this.getAnAccess() } + /** + * Holds if this variable is `constexpr`. + */ + predicate isConstexpr() { + this.hasSpecifier("is_constexpr") + } + /** * Holds if this variable is constructed from `v` as a result * of template instantiation. If so, it originates either from a template diff --git a/cpp/ql/test/library-tests/variables/constexpr/constexpr.cpp b/cpp/ql/test/library-tests/variables/constexpr/constexpr.cpp new file mode 100644 index 000000000000..63def166619f --- /dev/null +++ b/cpp/ql/test/library-tests/variables/constexpr/constexpr.cpp @@ -0,0 +1,6 @@ + +constexpr int var_constexpr = 5; +int var_not_constexpr_initialised = 6; +const int var_not_constexpr_const = 7; +int var_not_constexpr; + diff --git a/cpp/ql/test/library-tests/variables/constexpr/constexpr.expected b/cpp/ql/test/library-tests/variables/constexpr/constexpr.expected new file mode 100644 index 000000000000..04efceae7103 --- /dev/null +++ b/cpp/ql/test/library-tests/variables/constexpr/constexpr.expected @@ -0,0 +1,10 @@ +| constexpr.cpp:2:15:2:27 | var_constexpr | true | +| constexpr.cpp:3:5:3:33 | var_not_constexpr_initialised | false | +| constexpr.cpp:4:11:4:33 | var_not_constexpr_const | false | +| constexpr.cpp:5:5:5:21 | var_not_constexpr | false | +| file://:0:0:0:0 | fp_offset | false | +| file://:0:0:0:0 | gp_offset | false | +| file://:0:0:0:0 | overflow_arg_area | false | +| file://:0:0:0:0 | p#0 | false | +| file://:0:0:0:0 | p#0 | false | +| file://:0:0:0:0 | reg_save_area | false | diff --git a/cpp/ql/test/library-tests/variables/constexpr/constexpr.ql b/cpp/ql/test/library-tests/variables/constexpr/constexpr.ql new file mode 100644 index 000000000000..1bb474de828e --- /dev/null +++ b/cpp/ql/test/library-tests/variables/constexpr/constexpr.ql @@ -0,0 +1,5 @@ +import cpp + +from Variable v +select v, + any(boolean b | if v.isConstexpr() then b = true else b = false) From 2dc7f32ca34057ae161dbc06ed40dfcc157d5015 Mon Sep 17 00:00:00 2001 From: Asger F Date: Thu, 28 Feb 2019 15:27:49 +0000 Subject: [PATCH 51/71] JS: add Express to list of updated frameworks --- change-notes/1.20/analysis-javascript.md | 1 + 1 file changed, 1 insertion(+) diff --git a/change-notes/1.20/analysis-javascript.md b/change-notes/1.20/analysis-javascript.md index 13f6af667009..c5710b77b3cd 100644 --- a/change-notes/1.20/analysis-javascript.md +++ b/change-notes/1.20/analysis-javascript.md @@ -5,6 +5,7 @@ * Support for many frameworks and libraries has been improved, in particular including the following: - [a-sync-waterfall](https://www.npmjs.com/package/a-sync-waterfall) - [Electron](https://electronjs.org) + - [Express](https://npmjs.org/express) - [hapi](https://hapijs.com/) - [js-cookie](https://github.com/js-cookie/js-cookie) - [React](https://reactjs.org/) From 2bfb015218f9f67d1720858c8c854ad848692b39 Mon Sep 17 00:00:00 2001 From: Asger F Date: Thu, 28 Feb 2019 16:47:53 +0000 Subject: [PATCH 52/71] JS: Add closure string ops --- .../ql/src/semmle/javascript/StringOps.qll | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/StringOps.qll b/javascript/ql/src/semmle/javascript/StringOps.qll index 4b45a965c2dc..f9818cd46fa9 100644 --- a/javascript/ql/src/semmle/javascript/StringOps.qll +++ b/javascript/ql/src/semmle/javascript/StringOps.qll @@ -112,14 +112,18 @@ module StringOps { } /** - * A call of form `_.startsWith(A, B)` or `ramda.startsWith(A, B)`. + * A call of form `_.startsWith(A, B)` or `ramda.startsWith(A, B)` or `goog.string.startsWith(A, B)`. */ private class StartsWith_Library extends Range, DataFlow::CallNode { StartsWith_Library() { getNumArgument() = 2 and exists(DataFlow::SourceNode callee | this = callee.getACall() | callee = LodashUnderscore::member("startsWith") or - callee = DataFlow::moduleMember("ramda", "startsWith") + callee = DataFlow::moduleMember("ramda", "startsWith") or + exists(string name | + callee = Closure::moduleImport("goog.string." + name) and + (name = "startsWith" or name = "caseInsensitiveStartsWith") + ) ) } @@ -250,6 +254,9 @@ module StringOps { exists(string name | this = LodashUnderscore::member(name).getACall() and (name = "includes" or name = "include" or name = "contains") + or + this = Closure::moduleImport("goog.string." + name).getACall() and + (name = "contains" or name = "caseInsensitiveContains") ) } @@ -416,7 +423,11 @@ module StringOps { getNumArgument() = 2 and exists(DataFlow::SourceNode callee | this = callee.getACall() | callee = LodashUnderscore::member("endsWith") or - callee = DataFlow::moduleMember("ramda", "endsWith") + callee = DataFlow::moduleMember("ramda", "endsWith") or + exists(string name | + callee = Closure::moduleImport("goog.string." + name) and + (name = "endsWith" or name = "caseInsensitiveEndsWith") + ) ) } From 47b5f34870d234864604ce523a73d81593f16795 Mon Sep 17 00:00:00 2001 From: Asger F Date: Thu, 28 Feb 2019 16:48:47 +0000 Subject: [PATCH 53/71] JS: shift line numbers in test output --- .../StringOps/EndsWith/EndsWith.expected | 6 ++-- .../library-tests/StringOps/EndsWith/tst.js | 1 + .../StringOps/Includes/Includes.expected | 14 +++++----- .../library-tests/StringOps/Includes/tst.js | 1 + .../StringOps/StartsWith/StartsWith.expected | 28 +++++++++---------- .../library-tests/StringOps/StartsWith/tst.js | 1 + 6 files changed, 27 insertions(+), 24 deletions(-) diff --git a/javascript/ql/test/library-tests/StringOps/EndsWith/EndsWith.expected b/javascript/ql/test/library-tests/StringOps/EndsWith/EndsWith.expected index 210b525c5609..a5cc0fc6c6af 100644 --- a/javascript/ql/test/library-tests/StringOps/EndsWith/EndsWith.expected +++ b/javascript/ql/test/library-tests/StringOps/EndsWith/EndsWith.expected @@ -1,3 +1,3 @@ -| tst.js:5:7:5:19 | A.endsWith(B) | tst.js:5:7:5:7 | A | tst.js:5:18:5:18 | B | true | -| tst.js:6:7:6:22 | _.endsWith(A, B) | tst.js:6:18:6:18 | A | tst.js:6:21:6:21 | B | true | -| tst.js:7:7:7:22 | R.endsWith(A, B) | tst.js:7:18:7:18 | A | tst.js:7:21:7:21 | B | true | +| tst.js:6:7:6:19 | A.endsWith(B) | tst.js:6:7:6:7 | A | tst.js:6:18:6:18 | B | true | +| tst.js:7:7:7:22 | _.endsWith(A, B) | tst.js:7:18:7:18 | A | tst.js:7:21:7:21 | B | true | +| tst.js:8:7:8:22 | R.endsWith(A, B) | tst.js:8:18:8:18 | A | tst.js:8:21:8:21 | B | true | diff --git a/javascript/ql/test/library-tests/StringOps/EndsWith/tst.js b/javascript/ql/test/library-tests/StringOps/EndsWith/tst.js index 149194521fd0..9ed8fedd47c7 100644 --- a/javascript/ql/test/library-tests/StringOps/EndsWith/tst.js +++ b/javascript/ql/test/library-tests/StringOps/EndsWith/tst.js @@ -1,6 +1,7 @@ import * as _ from 'underscore'; import * as R from 'ramda'; + function test() { if (A.endsWith(B)) {} if (_.endsWith(A, B)) {} diff --git a/javascript/ql/test/library-tests/StringOps/Includes/Includes.expected b/javascript/ql/test/library-tests/StringOps/Includes/Includes.expected index 2567e4e6abe3..07dbc9ab74ca 100644 --- a/javascript/ql/test/library-tests/StringOps/Includes/Includes.expected +++ b/javascript/ql/test/library-tests/StringOps/Includes/Includes.expected @@ -1,7 +1,7 @@ -| tst.js:4:7:4:19 | A.includes(B) | tst.js:4:7:4:7 | A | tst.js:4:18:4:18 | B | true | -| tst.js:5:7:5:22 | _.includes(A, B) | tst.js:5:18:5:18 | A | tst.js:5:21:5:21 | B | true | -| tst.js:6:7:6:25 | A.indexOf(B) !== -1 | tst.js:6:7:6:7 | A | tst.js:6:17:6:17 | B | true | -| tst.js:7:7:7:23 | A.indexOf(B) >= 0 | tst.js:7:7:7:7 | A | tst.js:7:17:7:17 | B | true | -| tst.js:8:7:8:19 | ~A.indexOf(B) | tst.js:8:8:8:8 | A | tst.js:8:18:8:18 | B | true | -| tst.js:11:7:11:25 | A.indexOf(B) === -1 | tst.js:11:7:11:7 | A | tst.js:11:17:11:17 | B | false | -| tst.js:12:7:12:22 | A.indexOf(B) < 0 | tst.js:12:7:12:7 | A | tst.js:12:17:12:17 | B | false | +| tst.js:5:7:5:19 | A.includes(B) | tst.js:5:7:5:7 | A | tst.js:5:18:5:18 | B | true | +| tst.js:6:7:6:22 | _.includes(A, B) | tst.js:6:18:6:18 | A | tst.js:6:21:6:21 | B | true | +| tst.js:7:7:7:25 | A.indexOf(B) !== -1 | tst.js:7:7:7:7 | A | tst.js:7:17:7:17 | B | true | +| tst.js:8:7:8:23 | A.indexOf(B) >= 0 | tst.js:8:7:8:7 | A | tst.js:8:17:8:17 | B | true | +| tst.js:9:7:9:19 | ~A.indexOf(B) | tst.js:9:8:9:8 | A | tst.js:9:18:9:18 | B | true | +| tst.js:12:7:12:25 | A.indexOf(B) === -1 | tst.js:12:7:12:7 | A | tst.js:12:17:12:17 | B | false | +| tst.js:13:7:13:22 | A.indexOf(B) < 0 | tst.js:13:7:13:7 | A | tst.js:13:17:13:17 | B | false | diff --git a/javascript/ql/test/library-tests/StringOps/Includes/tst.js b/javascript/ql/test/library-tests/StringOps/Includes/tst.js index e0cecf8a54db..257439ced10e 100644 --- a/javascript/ql/test/library-tests/StringOps/Includes/tst.js +++ b/javascript/ql/test/library-tests/StringOps/Includes/tst.js @@ -1,5 +1,6 @@ import * as _ from 'lodash'; + function test() { if (A.includes(B)) {} if (_.includes(A, B)) {} diff --git a/javascript/ql/test/library-tests/StringOps/StartsWith/StartsWith.expected b/javascript/ql/test/library-tests/StringOps/StartsWith/StartsWith.expected index e139bfccbfd8..fa62bdc947ba 100644 --- a/javascript/ql/test/library-tests/StringOps/StartsWith/StartsWith.expected +++ b/javascript/ql/test/library-tests/StringOps/StartsWith/StartsWith.expected @@ -1,14 +1,14 @@ -| tst.js:5:9:5:23 | A.startsWith(B) | tst.js:5:9:5:9 | A | tst.js:5:22:5:22 | B | true | -| tst.js:6:9:6:26 | _.startsWith(A, B) | tst.js:6:22:6:22 | A | tst.js:6:25:6:25 | B | true | -| tst.js:7:9:7:26 | R.startsWith(A, B) | tst.js:7:22:7:22 | A | tst.js:7:25:7:25 | B | true | -| tst.js:8:9:8:26 | A.indexOf(B) === 0 | tst.js:8:9:8:9 | A | tst.js:8:19:8:19 | B | true | -| tst.js:9:9:9:26 | A.indexOf(B) !== 0 | tst.js:9:9:9:9 | A | tst.js:9:19:9:19 | B | false | -| tst.js:10:9:10:26 | 0 !== A.indexOf(B) | tst.js:10:15:10:15 | A | tst.js:10:25:10:25 | B | false | -| tst.js:11:9:11:25 | 0 != A.indexOf(B) | tst.js:11:14:11:14 | A | tst.js:11:24:11:24 | B | false | -| tst.js:12:9:12:20 | A.indexOf(B) | tst.js:12:9:12:9 | A | tst.js:12:19:12:19 | B | false | -| tst.js:13:10:13:21 | A.indexOf(B) | tst.js:13:10:13:10 | A | tst.js:13:20:13:20 | B | false | -| tst.js:14:11:14:22 | A.indexOf(B) | tst.js:14:11:14:11 | A | tst.js:14:21:14:21 | B | false | -| tst.js:15:9:15:38 | A.subst ... ) === B | tst.js:15:9:15:9 | A | tst.js:15:38:15:38 | B | true | -| tst.js:16:9:16:38 | A.subst ... ) !== B | tst.js:16:9:16:9 | A | tst.js:16:38:16:38 | B | false | -| tst.js:17:9:17:35 | A.subst ... ) === B | tst.js:17:9:17:9 | A | tst.js:17:35:17:35 | B | true | -| tst.js:18:9:18:36 | A.subst ... "web/" | tst.js:18:9:18:9 | A | tst.js:18:31:18:36 | "web/" | true | +| tst.js:6:9:6:23 | A.startsWith(B) | tst.js:6:9:6:9 | A | tst.js:6:22:6:22 | B | true | +| tst.js:7:9:7:26 | _.startsWith(A, B) | tst.js:7:22:7:22 | A | tst.js:7:25:7:25 | B | true | +| tst.js:8:9:8:26 | R.startsWith(A, B) | tst.js:8:22:8:22 | A | tst.js:8:25:8:25 | B | true | +| tst.js:9:9:9:26 | A.indexOf(B) === 0 | tst.js:9:9:9:9 | A | tst.js:9:19:9:19 | B | true | +| tst.js:10:9:10:26 | A.indexOf(B) !== 0 | tst.js:10:9:10:9 | A | tst.js:10:19:10:19 | B | false | +| tst.js:11:9:11:26 | 0 !== A.indexOf(B) | tst.js:11:15:11:15 | A | tst.js:11:25:11:25 | B | false | +| tst.js:12:9:12:25 | 0 != A.indexOf(B) | tst.js:12:14:12:14 | A | tst.js:12:24:12:24 | B | false | +| tst.js:13:9:13:20 | A.indexOf(B) | tst.js:13:9:13:9 | A | tst.js:13:19:13:19 | B | false | +| tst.js:14:10:14:21 | A.indexOf(B) | tst.js:14:10:14:10 | A | tst.js:14:20:14:20 | B | false | +| tst.js:15:11:15:22 | A.indexOf(B) | tst.js:15:11:15:11 | A | tst.js:15:21:15:21 | B | false | +| tst.js:16:9:16:38 | A.subst ... ) === B | tst.js:16:9:16:9 | A | tst.js:16:38:16:38 | B | true | +| tst.js:17:9:17:38 | A.subst ... ) !== B | tst.js:17:9:17:9 | A | tst.js:17:38:17:38 | B | false | +| tst.js:18:9:18:35 | A.subst ... ) === B | tst.js:18:9:18:9 | A | tst.js:18:35:18:35 | B | true | +| tst.js:19:9:19:36 | A.subst ... "web/" | tst.js:19:9:19:9 | A | tst.js:19:31:19:36 | "web/" | true | diff --git a/javascript/ql/test/library-tests/StringOps/StartsWith/tst.js b/javascript/ql/test/library-tests/StringOps/StartsWith/tst.js index 3a64cd8bbfd6..2a6c7c33c7c1 100644 --- a/javascript/ql/test/library-tests/StringOps/StartsWith/tst.js +++ b/javascript/ql/test/library-tests/StringOps/StartsWith/tst.js @@ -1,6 +1,7 @@ import * as _ from 'lodash'; import * as R from 'ramda'; + function f(A, B) { if (A.startsWith(B)) {} if (_.startsWith(A, B)) {} From 8dfec58428a8d76042db6eba4ee9686fd7921a52 Mon Sep 17 00:00:00 2001 From: Asger F Date: Thu, 28 Feb 2019 16:49:35 +0000 Subject: [PATCH 54/71] JS: Update test --- .../test/library-tests/StringOps/EndsWith/EndsWith.expected | 2 ++ javascript/ql/test/library-tests/StringOps/EndsWith/tst.js | 4 +++- .../test/library-tests/StringOps/Includes/Includes.expected | 2 ++ javascript/ql/test/library-tests/StringOps/Includes/tst.js | 5 ++++- .../library-tests/StringOps/StartsWith/StartsWith.expected | 2 ++ javascript/ql/test/library-tests/StringOps/StartsWith/tst.js | 5 ++++- 6 files changed, 17 insertions(+), 3 deletions(-) diff --git a/javascript/ql/test/library-tests/StringOps/EndsWith/EndsWith.expected b/javascript/ql/test/library-tests/StringOps/EndsWith/EndsWith.expected index a5cc0fc6c6af..58c0250f4dc1 100644 --- a/javascript/ql/test/library-tests/StringOps/EndsWith/EndsWith.expected +++ b/javascript/ql/test/library-tests/StringOps/EndsWith/EndsWith.expected @@ -1,3 +1,5 @@ | tst.js:6:7:6:19 | A.endsWith(B) | tst.js:6:7:6:7 | A | tst.js:6:18:6:18 | B | true | | tst.js:7:7:7:22 | _.endsWith(A, B) | tst.js:7:18:7:18 | A | tst.js:7:21:7:21 | B | true | | tst.js:8:7:8:22 | R.endsWith(A, B) | tst.js:8:18:8:18 | A | tst.js:8:21:8:21 | B | true | +| tst.js:9:7:9:28 | strings ... h(A, B) | tst.js:9:24:9:24 | A | tst.js:9:27:9:27 | B | true | +| tst.js:10:7:10:43 | strings ... h(A, B) | tst.js:10:39:10:39 | A | tst.js:10:42:10:42 | B | true | diff --git a/javascript/ql/test/library-tests/StringOps/EndsWith/tst.js b/javascript/ql/test/library-tests/StringOps/EndsWith/tst.js index 9ed8fedd47c7..9f0ce1df0ef5 100644 --- a/javascript/ql/test/library-tests/StringOps/EndsWith/tst.js +++ b/javascript/ql/test/library-tests/StringOps/EndsWith/tst.js @@ -1,9 +1,11 @@ import * as _ from 'underscore'; import * as R from 'ramda'; - +let strings = goog.require('goog.string'); function test() { if (A.endsWith(B)) {} if (_.endsWith(A, B)) {} if (R.endsWith(A, B)) {} + if (strings.endsWith(A, B)) {} + if (strings.caseInsensitiveEndsWith(A, B)) {} } diff --git a/javascript/ql/test/library-tests/StringOps/Includes/Includes.expected b/javascript/ql/test/library-tests/StringOps/Includes/Includes.expected index 07dbc9ab74ca..d5b7dcfdd183 100644 --- a/javascript/ql/test/library-tests/StringOps/Includes/Includes.expected +++ b/javascript/ql/test/library-tests/StringOps/Includes/Includes.expected @@ -5,3 +5,5 @@ | tst.js:9:7:9:19 | ~A.indexOf(B) | tst.js:9:8:9:8 | A | tst.js:9:18:9:18 | B | true | | tst.js:12:7:12:25 | A.indexOf(B) === -1 | tst.js:12:7:12:7 | A | tst.js:12:17:12:17 | B | false | | tst.js:13:7:13:22 | A.indexOf(B) < 0 | tst.js:13:7:13:7 | A | tst.js:13:17:13:17 | B | false | +| tst.js:20:7:20:28 | strings ... s(A, B) | tst.js:20:24:20:24 | A | tst.js:20:27:20:27 | B | true | +| tst.js:21:7:21:43 | strings ... s(A, B) | tst.js:21:39:21:39 | A | tst.js:21:42:21:42 | B | true | diff --git a/javascript/ql/test/library-tests/StringOps/Includes/tst.js b/javascript/ql/test/library-tests/StringOps/Includes/tst.js index 257439ced10e..fada4e1de29f 100644 --- a/javascript/ql/test/library-tests/StringOps/Includes/tst.js +++ b/javascript/ql/test/library-tests/StringOps/Includes/tst.js @@ -1,5 +1,5 @@ import * as _ from 'lodash'; - +let strings = goog.require('goog.string'); function test() { if (A.includes(B)) {} @@ -16,4 +16,7 @@ function test() { if (A.indexOf(B) === 0) {} if (A.indexOf(B) !== 0) {} if (A.indexOf(B) > 0) {} + + if (strings.contains(A, B)) {} + if (strings.caseInsensitiveContains(A, B)) {} } diff --git a/javascript/ql/test/library-tests/StringOps/StartsWith/StartsWith.expected b/javascript/ql/test/library-tests/StringOps/StartsWith/StartsWith.expected index fa62bdc947ba..3910799bd75a 100644 --- a/javascript/ql/test/library-tests/StringOps/StartsWith/StartsWith.expected +++ b/javascript/ql/test/library-tests/StringOps/StartsWith/StartsWith.expected @@ -12,3 +12,5 @@ | tst.js:17:9:17:38 | A.subst ... ) !== B | tst.js:17:9:17:9 | A | tst.js:17:38:17:38 | B | false | | tst.js:18:9:18:35 | A.subst ... ) === B | tst.js:18:9:18:9 | A | tst.js:18:35:18:35 | B | true | | tst.js:19:9:19:36 | A.subst ... "web/" | tst.js:19:9:19:9 | A | tst.js:19:31:19:36 | "web/" | true | +| tst.js:32:9:32:32 | strings ... h(A, B) | tst.js:32:28:32:28 | A | tst.js:32:31:32:31 | B | true | +| tst.js:33:9:33:47 | strings ... h(A, B) | tst.js:33:43:33:43 | A | tst.js:33:46:33:46 | B | true | diff --git a/javascript/ql/test/library-tests/StringOps/StartsWith/tst.js b/javascript/ql/test/library-tests/StringOps/StartsWith/tst.js index 2a6c7c33c7c1..97bbf240ec62 100644 --- a/javascript/ql/test/library-tests/StringOps/StartsWith/tst.js +++ b/javascript/ql/test/library-tests/StringOps/StartsWith/tst.js @@ -1,6 +1,6 @@ import * as _ from 'lodash'; import * as R from 'ramda'; - +let strings = goog.require('goog.string'); function f(A, B) { if (A.startsWith(B)) {} @@ -28,4 +28,7 @@ function f(A, B) { if (A.indexOf(B, 2)) {} if (~A.indexOf(B)) {} // checks for existence, not startsWith if (A.substring(B.length) === 0) {} + + if (strings.startsWith(A, B)) {} + if (strings.caseInsensitiveStartsWith(A, B)) {} } From baa4f0825991a1b22a64af3b71fc3b8eb0013fa5 Mon Sep 17 00:00:00 2001 From: Jason Reed Date: Mon, 11 Feb 2019 10:02:36 -0500 Subject: [PATCH 55/71] JS: Add new query for ZipSlip (CWE-022) --- .../ql/src/Security/CWE-022/ZipSlip.qhelp | 38 +++++++++ javascript/ql/src/Security/CWE-022/ZipSlip.ql | 22 +++++ .../ql/src/Security/CWE-022/ZipSlipBad.js | 9 ++ .../javascript/security/dataflow/ZipSlip.qll | 84 +++++++++++++++++++ .../Security/CWE-022/ZipSlip/ZipSlip.expected | 5 ++ .../Security/CWE-022/ZipSlip/ZipSlip.qlref | 1 + .../Security/CWE-022/ZipSlip/ZipSlipBad.js | 9 ++ .../Security/CWE-022/ZipSlip/ZipSlipGood.js | 14 ++++ 8 files changed, 182 insertions(+) create mode 100644 javascript/ql/src/Security/CWE-022/ZipSlip.qhelp create mode 100644 javascript/ql/src/Security/CWE-022/ZipSlip.ql create mode 100644 javascript/ql/src/Security/CWE-022/ZipSlipBad.js create mode 100644 javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll create mode 100644 javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.expected create mode 100644 javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.qlref create mode 100644 javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipBad.js create mode 100644 javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipGood.js diff --git a/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp b/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp new file mode 100644 index 000000000000..c595ceb7739a --- /dev/null +++ b/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp @@ -0,0 +1,38 @@ + + + + +

      Extracting files from a malicious zip archive without validating that the destination file path +is within the destination directory can cause files outside the destination directory to be +overwritten, due to the possible presence of directory traversal elements (..) in +archive paths.

      + +

      Zip archives contain archive entries representing each file in the archive. These entries +include a file path for the entry, but these file paths are not restricted and may contain +unexpected special elements such as the directory traversal element (..). If these +file paths are used to determine an output file to write the contents of the archive item to, then +the file may be written to an unexpected location. This can result in sensitive information being +revealed or deleted, or an attacker being able to influence behavior by modifying unexpected +files.

      + + + + +

      Ensure that output paths constructed from zip archive entries are validated +to prevent writing files to unexpected locations.

      + +       + + +

      +Here is an example of extracting an archive without validating +filenames. If archive.zip contained relative paths (for +instance, if it were created by something like zip archive.zip +../file.txt) then executing this code would write to those paths. +

      + + +       +       diff --git a/javascript/ql/src/Security/CWE-022/ZipSlip.ql b/javascript/ql/src/Security/CWE-022/ZipSlip.ql new file mode 100644 index 000000000000..6c6466b64ab9 --- /dev/null +++ b/javascript/ql/src/Security/CWE-022/ZipSlip.ql @@ -0,0 +1,22 @@ +/** + * @name Arbitrary file write during zip extraction ("Zip Slip") + * @description Extracting files from a malicious zip archive without validating that the + * destination file path is within the destination directory can cause files outside + * the destination directory to be overwritten. + * @kind path-problem + * @id cs/zipslip + * @problem.severity error + * @precision high + * @tags security + * external/cwe/cwe-022 + */ + +import javascript +import semmle.javascript.security.dataflow.ZipSlip::ZipSlip +import DataFlow::PathGraph + +from Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink +where cfg.hasFlowPath(source, sink) +select sink.getNode(), source, sink, + "Unsanitized zip archive $@, which may contain '..', is used in a file system operation.", + source.getNode(), "item path" \ No newline at end of file diff --git a/javascript/ql/src/Security/CWE-022/ZipSlipBad.js b/javascript/ql/src/Security/CWE-022/ZipSlipBad.js new file mode 100644 index 000000000000..fc6ac2514485 --- /dev/null +++ b/javascript/ql/src/Security/CWE-022/ZipSlipBad.js @@ -0,0 +1,9 @@ +const fs = require('fs'); +const unzip = require('unzip'); + +fs.createReadStream('archive.zip') + .pipe(unzip.Parse()) + .on('entry', entry => { + const fileName = entry.path; + entry.pipe(fs.createWriteStream(entry.path)); + }); diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll new file mode 100644 index 000000000000..43c00711a8b9 --- /dev/null +++ b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll @@ -0,0 +1,84 @@ +/** + * Provides a taint tracking configuration for reasoning about unsafe zip extraction. + */ + +import javascript + +module ZipSlip { + /** + * A data flow source for unsafe zip extraction. + */ + abstract class Source extends DataFlow::Node { } + + /** + * A data flow sink for unsafe zip extraction. + */ + abstract class Sink extends DataFlow::Node { } + + /** + * A sanitizer guard for unsafe zip extraction. + */ + abstract class SanitizerGuard extends + TaintTracking::SanitizerGuardNode, + DataFlow::ValueNode { } + + /** A taint tracking configuration for Zip Slip */ + class Configuration extends TaintTracking::Configuration { + Configuration() { this = "ZipSlip" } + + override predicate isSource(DataFlow::Node source) { source instanceof Source } + + override predicate isSink(DataFlow::Node sink) { sink instanceof Sink } + + override predicate isSanitizerGuard(TaintTracking::SanitizerGuardNode nd) { + nd instanceof SanitizerGuard + } + } + + /** + * An access to the filepath of an entry of a zipfile being extracted by + * npm module `unzip`. + */ + class UnzipEntrySource extends Source { + UnzipEntrySource() { + exists(DataFlow::MethodCallNode pipe, DataFlow::MethodCallNode on | + pipe.getMethodName() = "pipe" + and pipe.getArgument(0) = DataFlow::moduleImport("unzip").getAMemberCall("Parse") + and on = pipe.getAMemberCall("on") + and this = on.getCallback(1).getParameter(0).getAPropertyRead("path")) + } + } + + /** + * A sink that is the path that a createWriteStream gets created at. + * This is not covered by FileSystemWriteSink, because it is + * required that a write actually takes place to the stream. + * However, we want to consider even the bare createWriteStream to + * be a zipslip vulnerability since it may truncate an existing file. + */ + class CreateWriteStreamSink extends Sink { + CreateWriteStreamSink() { + this = DataFlow::moduleImport("fs").getAMemberCall("createWriteStream").getArgument(0) + } + } + + /** A sink that is a file path that gets written to. */ + class FileSystemWriteSink extends Sink { + FileSystemWriteSink() { + exists(FileSystemWriteAccess fsw | fsw.getAPathArgument() = this) + } + } + + /** A check that a path string does not include '..' */ + class NoParentDirSanitizerGuard extends SanitizerGuard { + StringOps::Includes incl; + + NoParentDirSanitizerGuard() { this = incl } + + override predicate sanitizes(boolean outcome, Expr e) { + incl.getPolarity().booleanNot() = outcome + and incl.getBaseString().asExpr() = e + and incl.getSubstring().mayHaveStringValue("..") + } + } +} diff --git a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.expected b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.expected new file mode 100644 index 000000000000..88b9ef8722e9 --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.expected @@ -0,0 +1,5 @@ +nodes +| ZipSlipBad.js:8:37:8:46 | entry.path | +edges +#select +| ZipSlipBad.js:8:37:8:46 | entry.path | ZipSlipBad.js:8:37:8:46 | entry.path | ZipSlipBad.js:8:37:8:46 | entry.path | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlipBad.js:8:37:8:46 | entry.path | item path | diff --git a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.qlref b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.qlref new file mode 100644 index 000000000000..0ac6382f48ab --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.qlref @@ -0,0 +1 @@ +Security/CWE-022/ZipSlip.ql diff --git a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipBad.js b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipBad.js new file mode 100644 index 000000000000..fc6ac2514485 --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipBad.js @@ -0,0 +1,9 @@ +const fs = require('fs'); +const unzip = require('unzip'); + +fs.createReadStream('archive.zip') + .pipe(unzip.Parse()) + .on('entry', entry => { + const fileName = entry.path; + entry.pipe(fs.createWriteStream(entry.path)); + }); diff --git a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipGood.js b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipGood.js new file mode 100644 index 000000000000..325e9586223f --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipGood.js @@ -0,0 +1,14 @@ +const fs = require('fs'); +const unzip = require('unzip'); + +fs.createReadStream('archive.zip') + .pipe(unzip.Parse()) + .on('entry', entry => { + const fileName = entry.path; + if (entry.path.indexOf('..') == -1) { + entry.pipe(fs.createWriteStream(entry.path)); + } + else { + console.log('skipping bad path', entry.path); + } + }); From abd2644af7b9b26f09b1af236decf65fdfb12276 Mon Sep 17 00:00:00 2001 From: Jason Reed Date: Wed, 13 Feb 2019 13:53:23 -0500 Subject: [PATCH 56/71] JS: Address review comments --- javascript/ql/src/Security/CWE-022/ZipSlip.ql | 2 +- .../javascript/security/dataflow/ZipSlip.qll | 27 ++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/javascript/ql/src/Security/CWE-022/ZipSlip.ql b/javascript/ql/src/Security/CWE-022/ZipSlip.ql index 6c6466b64ab9..44842ab70199 100644 --- a/javascript/ql/src/Security/CWE-022/ZipSlip.ql +++ b/javascript/ql/src/Security/CWE-022/ZipSlip.ql @@ -4,7 +4,7 @@ * destination file path is within the destination directory can cause files outside * the destination directory to be overwritten. * @kind path-problem - * @id cs/zipslip + * @id js/zipslip * @problem.severity error * @precision high * @tags security diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll index 43c00711a8b9..a7d42649d7bc 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll @@ -36,14 +36,25 @@ module ZipSlip { } /** - * An access to the filepath of an entry of a zipfile being extracted by - * npm module `unzip`. + * An access to the filepath of an entry of a zipfile being extracted + * by npm module `unzip`. For example, in + * ```javascript + * const unzip = require('unzip'); + * + * fs.createReadStream('archive.zip') + * .pipe(unzip.Parse()) + * .on('entry', entry => { + * const path = entry.path; + * }); + * ``` + * there is an `UnzipEntrySource` node corresponding to + * the expression `entry.path`. */ class UnzipEntrySource extends Source { UnzipEntrySource() { exists(DataFlow::MethodCallNode pipe, DataFlow::MethodCallNode on | pipe.getMethodName() = "pipe" - and pipe.getArgument(0) = DataFlow::moduleImport("unzip").getAMemberCall("Parse") + and pipe.getArgument(0).getALocalSource() = DataFlow::moduleImport("unzip").getAMemberCall("Parse") and on = pipe.getAMemberCall("on") and this = on.getCallback(1).getParameter(0).getAPropertyRead("path")) } @@ -69,6 +80,14 @@ module ZipSlip { } } + /** + * Gets a string which suffices to search for to ensure that a + * filepath will not refer to parent directories. + */ + string getAParentDirName() { + result = any(string s | s = ".." or s = "../") + } + /** A check that a path string does not include '..' */ class NoParentDirSanitizerGuard extends SanitizerGuard { StringOps::Includes incl; @@ -78,7 +97,7 @@ module ZipSlip { override predicate sanitizes(boolean outcome, Expr e) { incl.getPolarity().booleanNot() = outcome and incl.getBaseString().asExpr() = e - and incl.getSubstring().mayHaveStringValue("..") + and incl.getSubstring().mayHaveStringValue(getAParentDirName()) } } } From 32d48ba98b13899335042664683acb79a1c5000d Mon Sep 17 00:00:00 2001 From: Jason Reed Date: Wed, 13 Feb 2019 14:01:26 -0500 Subject: [PATCH 57/71] JS: Run auto-formatter --- javascript/ql/src/Security/CWE-022/ZipSlip.ql | 2 +- .../javascript/security/dataflow/ZipSlip.qll | 30 ++++++++----------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/javascript/ql/src/Security/CWE-022/ZipSlip.ql b/javascript/ql/src/Security/CWE-022/ZipSlip.ql index 44842ab70199..e9c92163d055 100644 --- a/javascript/ql/src/Security/CWE-022/ZipSlip.ql +++ b/javascript/ql/src/Security/CWE-022/ZipSlip.ql @@ -19,4 +19,4 @@ from Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink where cfg.hasFlowPath(source, sink) select sink.getNode(), source, sink, "Unsanitized zip archive $@, which may contain '..', is used in a file system operation.", - source.getNode(), "item path" \ No newline at end of file + source.getNode(), "item path" diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll index a7d42649d7bc..1ab50d459bb8 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll @@ -18,9 +18,7 @@ module ZipSlip { /** * A sanitizer guard for unsafe zip extraction. */ - abstract class SanitizerGuard extends - TaintTracking::SanitizerGuardNode, - DataFlow::ValueNode { } + abstract class SanitizerGuard extends TaintTracking::SanitizerGuardNode, DataFlow::ValueNode { } /** A taint tracking configuration for Zip Slip */ class Configuration extends TaintTracking::Configuration { @@ -53,10 +51,12 @@ module ZipSlip { class UnzipEntrySource extends Source { UnzipEntrySource() { exists(DataFlow::MethodCallNode pipe, DataFlow::MethodCallNode on | - pipe.getMethodName() = "pipe" - and pipe.getArgument(0).getALocalSource() = DataFlow::moduleImport("unzip").getAMemberCall("Parse") - and on = pipe.getAMemberCall("on") - and this = on.getCallback(1).getParameter(0).getAPropertyRead("path")) + pipe.getMethodName() = "pipe" and + pipe.getArgument(0).getALocalSource() = DataFlow::moduleImport("unzip") + .getAMemberCall("Parse") and + on = pipe.getAMemberCall("on") and + this = on.getCallback(1).getParameter(0).getAPropertyRead("path") + ) } } @@ -75,29 +75,25 @@ module ZipSlip { /** A sink that is a file path that gets written to. */ class FileSystemWriteSink extends Sink { - FileSystemWriteSink() { - exists(FileSystemWriteAccess fsw | fsw.getAPathArgument() = this) - } + FileSystemWriteSink() { exists(FileSystemWriteAccess fsw | fsw.getAPathArgument() = this) } } /** * Gets a string which suffices to search for to ensure that a * filepath will not refer to parent directories. */ - string getAParentDirName() { - result = any(string s | s = ".." or s = "../") - } + string getAParentDirName() { result = any(string s | s = ".." or s = "../") } /** A check that a path string does not include '..' */ class NoParentDirSanitizerGuard extends SanitizerGuard { StringOps::Includes incl; - NoParentDirSanitizerGuard() { this = incl } + NoParentDirSanitizerGuard() { this = incl } override predicate sanitizes(boolean outcome, Expr e) { - incl.getPolarity().booleanNot() = outcome - and incl.getBaseString().asExpr() = e - and incl.getSubstring().mayHaveStringValue(getAParentDirName()) + incl.getPolarity().booleanNot() = outcome and + incl.getBaseString().asExpr() = e and + incl.getSubstring().mayHaveStringValue(getAParentDirName()) } } } From 23d37c716713e0ec93acc6f80892a1824ef49d2e Mon Sep 17 00:00:00 2001 From: Jason Reed Date: Thu, 14 Feb 2019 13:25:09 -0500 Subject: [PATCH 58/71] JS: Unbreak TaintedPath --- .../Security/CWE-022/{ => TaintedPath}/TaintedPath-es6.js | 0 .../Security/CWE-022/{ => TaintedPath}/TaintedPath.expected | 0 .../query-tests/Security/CWE-022/{ => TaintedPath}/TaintedPath.js | 0 .../Security/CWE-022/{ => TaintedPath}/TaintedPath.qlref | 0 .../ql/test/query-tests/Security/CWE-022/{ => TaintedPath}/fs.js | 0 .../Security/CWE-022/{ => TaintedPath}/tainted-array-steps.js | 0 .../Security/CWE-022/{ => TaintedPath}/tainted-require.js | 0 .../Security/CWE-022/{ => TaintedPath}/tainted-sendFile.js | 0 .../test/query-tests/Security/CWE-022/{ => TaintedPath}/views.js | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename javascript/ql/test/query-tests/Security/CWE-022/{ => TaintedPath}/TaintedPath-es6.js (100%) rename javascript/ql/test/query-tests/Security/CWE-022/{ => TaintedPath}/TaintedPath.expected (100%) rename javascript/ql/test/query-tests/Security/CWE-022/{ => TaintedPath}/TaintedPath.js (100%) rename javascript/ql/test/query-tests/Security/CWE-022/{ => TaintedPath}/TaintedPath.qlref (100%) rename javascript/ql/test/query-tests/Security/CWE-022/{ => TaintedPath}/fs.js (100%) rename javascript/ql/test/query-tests/Security/CWE-022/{ => TaintedPath}/tainted-array-steps.js (100%) rename javascript/ql/test/query-tests/Security/CWE-022/{ => TaintedPath}/tainted-require.js (100%) rename javascript/ql/test/query-tests/Security/CWE-022/{ => TaintedPath}/tainted-sendFile.js (100%) rename javascript/ql/test/query-tests/Security/CWE-022/{ => TaintedPath}/views.js (100%) diff --git a/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath-es6.js b/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/TaintedPath-es6.js similarity index 100% rename from javascript/ql/test/query-tests/Security/CWE-022/TaintedPath-es6.js rename to javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/TaintedPath-es6.js diff --git a/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath.expected b/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/TaintedPath.expected similarity index 100% rename from javascript/ql/test/query-tests/Security/CWE-022/TaintedPath.expected rename to javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/TaintedPath.expected diff --git a/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath.js b/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/TaintedPath.js similarity index 100% rename from javascript/ql/test/query-tests/Security/CWE-022/TaintedPath.js rename to javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/TaintedPath.js diff --git a/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath.qlref b/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/TaintedPath.qlref similarity index 100% rename from javascript/ql/test/query-tests/Security/CWE-022/TaintedPath.qlref rename to javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/TaintedPath.qlref diff --git a/javascript/ql/test/query-tests/Security/CWE-022/fs.js b/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/fs.js similarity index 100% rename from javascript/ql/test/query-tests/Security/CWE-022/fs.js rename to javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/fs.js diff --git a/javascript/ql/test/query-tests/Security/CWE-022/tainted-array-steps.js b/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/tainted-array-steps.js similarity index 100% rename from javascript/ql/test/query-tests/Security/CWE-022/tainted-array-steps.js rename to javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/tainted-array-steps.js diff --git a/javascript/ql/test/query-tests/Security/CWE-022/tainted-require.js b/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/tainted-require.js similarity index 100% rename from javascript/ql/test/query-tests/Security/CWE-022/tainted-require.js rename to javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/tainted-require.js diff --git a/javascript/ql/test/query-tests/Security/CWE-022/tainted-sendFile.js b/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/tainted-sendFile.js similarity index 100% rename from javascript/ql/test/query-tests/Security/CWE-022/tainted-sendFile.js rename to javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/tainted-sendFile.js diff --git a/javascript/ql/test/query-tests/Security/CWE-022/views.js b/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/views.js similarity index 100% rename from javascript/ql/test/query-tests/Security/CWE-022/views.js rename to javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/views.js From b0636dd41067f330cf93991f872072bc78f8909e Mon Sep 17 00:00:00 2001 From: Jason Reed Date: Fri, 22 Feb 2019 09:56:44 -0500 Subject: [PATCH 59/71] JS: Better local flow through `.pipe` chaining --- .../javascript/security/dataflow/ZipSlip.qll | 37 ++++++++++++++++--- .../Security/CWE-022/ZipSlip/ZipSlip.expected | 8 ++++ .../Security/CWE-022/ZipSlip/ZipSlipBad2.js | 8 ++++ .../Security/CWE-022/ZipSlip/externs.js | 11 ++++++ 4 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipBad2.js create mode 100644 javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/externs.js diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll index 1ab50d459bb8..bcf388ffaa02 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll @@ -33,6 +33,33 @@ module ZipSlip { } } + /** + * Holds if `node1` flows to `node2` in one step by virtue of + * `node2` being of the form `.pipe(node1)`. The reason this flow + * exists is that `.pipe` returns its argument to make chained + * stream operations work. + */ + predicate pipeStep(DataFlow::Node node1, DataFlow::MethodCallNode node2) { + node2.getMethodName() = "pipe" and + node1 = node2.getArgument(0) + } + + /** + * Holds if `node1` flows to `node2` in one step including the assumption that + * `x` flows to `.pipe(x)` + */ + predicate stepsThroughPipe(DataFlow::Node node1, DataFlow::Node node2) { + DataFlow::localFlowStep(node1, node2) or pipeStep(node1, node2) + } + + /** + * Holds if `node1` flows to `node2` including the assumption that + * `x` flows to `.pipe(x)` + */ + predicate flowsThroughPipe(DataFlow::Node node1, DataFlow::Node node2) { + stepsThroughPipe*(node1, node2) + } + /** * An access to the filepath of an entry of a zipfile being extracted * by npm module `unzip`. For example, in @@ -50,12 +77,10 @@ module ZipSlip { */ class UnzipEntrySource extends Source { UnzipEntrySource() { - exists(DataFlow::MethodCallNode pipe, DataFlow::MethodCallNode on | - pipe.getMethodName() = "pipe" and - pipe.getArgument(0).getALocalSource() = DataFlow::moduleImport("unzip") - .getAMemberCall("Parse") and - on = pipe.getAMemberCall("on") and - this = on.getCallback(1).getParameter(0).getAPropertyRead("path") + exists(DataFlow::SourceNode parsed | + flowsThroughPipe(DataFlow::moduleImport("unzip").getAMemberCall("Parse"), parsed) + and + this = parsed.getAMemberCall("on").getCallback(1).getParameter(0).getAPropertyRead("path") ) } } diff --git a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.expected b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.expected index 88b9ef8722e9..a9e316d7d469 100644 --- a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.expected +++ b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.expected @@ -1,5 +1,13 @@ nodes +| ZipSlipBad2.js:5:9:5:46 | fileName | +| ZipSlipBad2.js:5:20:5:46 | 'output ... ry.path | +| ZipSlipBad2.js:5:37:5:46 | entry.path | +| ZipSlipBad2.js:6:22:6:29 | fileName | | ZipSlipBad.js:8:37:8:46 | entry.path | edges +| ZipSlipBad2.js:5:9:5:46 | fileName | ZipSlipBad2.js:6:22:6:29 | fileName | +| ZipSlipBad2.js:5:20:5:46 | 'output ... ry.path | ZipSlipBad2.js:5:9:5:46 | fileName | +| ZipSlipBad2.js:5:37:5:46 | entry.path | ZipSlipBad2.js:5:20:5:46 | 'output ... ry.path | #select +| ZipSlipBad2.js:6:22:6:29 | fileName | ZipSlipBad2.js:5:37:5:46 | entry.path | ZipSlipBad2.js:6:22:6:29 | fileName | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlipBad2.js:5:37:5:46 | entry.path | item path | | ZipSlipBad.js:8:37:8:46 | entry.path | ZipSlipBad.js:8:37:8:46 | entry.path | ZipSlipBad.js:8:37:8:46 | entry.path | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlipBad.js:8:37:8:46 | entry.path | item path | diff --git a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipBad2.js b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipBad2.js new file mode 100644 index 000000000000..d582c680ef8e --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipBad2.js @@ -0,0 +1,8 @@ +var fs = require('fs'); +var unzip = require('unzip'); +fs.readFile('path/to/archive.zip', function (err, zipContents) { + unzip.Parse(zipContents).on('entry', function (entry) { + var fileName = 'output/path/' + entry.path; + fs.writeFileSync(fileName, entry.contents); + }); +}); diff --git a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/externs.js b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/externs.js new file mode 100644 index 000000000000..1a27aa787d74 --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/externs.js @@ -0,0 +1,11 @@ +/** + * @externs + */ +var fs = {}; + +/** + * @param {string} filename + * @param {*} data + * @return {void} + */ +fs.writeFileSync = function(filename, data) {}; From 09b9a577837ea99c91a974eedc5ade77cd15a1f5 Mon Sep 17 00:00:00 2001 From: Jason Reed Date: Mon, 25 Feb 2019 08:59:30 -0500 Subject: [PATCH 60/71] JS: More efficient reasoning through pipe --- .../javascript/security/dataflow/ZipSlip.qll | 40 +++++-------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll index bcf388ffaa02..f21c02e04c5e 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll @@ -34,30 +34,16 @@ module ZipSlip { } /** - * Holds if `node1` flows to `node2` in one step by virtue of - * `node2` being of the form `.pipe(node1)`. The reason this flow - * exists is that `.pipe` returns its argument to make chained - * stream operations work. + * Gets a node that can be a parsed zip archive. */ - predicate pipeStep(DataFlow::Node node1, DataFlow::MethodCallNode node2) { - node2.getMethodName() = "pipe" and - node1 = node2.getArgument(0) - } - - /** - * Holds if `node1` flows to `node2` in one step including the assumption that - * `x` flows to `.pipe(x)` - */ - predicate stepsThroughPipe(DataFlow::Node node1, DataFlow::Node node2) { - DataFlow::localFlowStep(node1, node2) or pipeStep(node1, node2) - } - - /** - * Holds if `node1` flows to `node2` including the assumption that - * `x` flows to `.pipe(x)` - */ - predicate flowsThroughPipe(DataFlow::Node node1, DataFlow::Node node2) { - stepsThroughPipe*(node1, node2) + DataFlow::SourceNode parsedArchive() { + result = DataFlow::moduleImport("unzip").getAMemberCall("Parse") + or + // `streamProducer.pipe(unzip.Parse())` is a typical (but not + // universal) pattern when using nodejs streams, whose return + // value is the parsed stream. + exists(DataFlow::MethodCallNode pipe | pipe.getMethodName() = "pipe" + and parsedArchive().flowsTo(pipe.getArgument(0))) } /** @@ -77,11 +63,7 @@ module ZipSlip { */ class UnzipEntrySource extends Source { UnzipEntrySource() { - exists(DataFlow::SourceNode parsed | - flowsThroughPipe(DataFlow::moduleImport("unzip").getAMemberCall("Parse"), parsed) - and - this = parsed.getAMemberCall("on").getCallback(1).getParameter(0).getAPropertyRead("path") - ) + this = parsedArchive().getAMemberCall("on").getCallback(1).getParameter(0).getAPropertyRead("path") } } @@ -107,7 +89,7 @@ module ZipSlip { * Gets a string which suffices to search for to ensure that a * filepath will not refer to parent directories. */ - string getAParentDirName() { result = any(string s | s = ".." or s = "../") } + string getAParentDirName() { result = ".." or result = "../" } /** A check that a path string does not include '..' */ class NoParentDirSanitizerGuard extends SanitizerGuard { From 2fc2a393b7052c60fab9bf4650f80b3ad2b6f09b Mon Sep 17 00:00:00 2001 From: Jason Reed Date: Tue, 26 Feb 2019 08:08:52 -0500 Subject: [PATCH 61/71] JS: Address review comments --- .../ql/src/Security/CWE-022/ZipSlip.qhelp | 18 +++++- javascript/ql/src/Security/CWE-022/ZipSlip.ql | 2 +- .../ql/src/Security/CWE-022/ZipSlipGood.js | 14 +++++ .../javascript/security/dataflow/ZipSlip.qll | 61 ++++++++++--------- 4 files changed, 64 insertions(+), 31 deletions(-) create mode 100644 javascript/ql/src/Security/CWE-022/ZipSlipGood.js diff --git a/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp b/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp index c595ceb7739a..07c4fb0c5790 100644 --- a/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp +++ b/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp @@ -16,13 +16,22 @@ file paths are used to determine an output file to write the contents of the arc the file may be written to an unexpected location. This can result in sensitive information being revealed or deleted, or an attacker being able to influence behavior by modifying unexpected files.

      - +

      For example, if a zip file contains a file entry ..\sneaky-file, and the zip file +is extracted to the directory c:\output, then naively combining the paths would result +in an output file path of c:\output\..\sneaky-file, which would cause the file to be +written to c:\sneaky-file.

      + +

      Ensure that output paths constructed from zip archive entries are validated to prevent writing files to unexpected locations.

      +

      The recommended way of writing an output file from a zip archive entry is to check that +".." does not occur in the path. +

      +       @@ -34,5 +43,12 @@ instance, if it were created by something like zip archive.zip

      + +

      To fix this vulnerability, we can to check that the path does not +contain any ".." in it. +

      + + +       diff --git a/javascript/ql/src/Security/CWE-022/ZipSlip.ql b/javascript/ql/src/Security/CWE-022/ZipSlip.ql index e9c92163d055..46b0269147cb 100644 --- a/javascript/ql/src/Security/CWE-022/ZipSlip.ql +++ b/javascript/ql/src/Security/CWE-022/ZipSlip.ql @@ -6,7 +6,7 @@ * @kind path-problem * @id js/zipslip * @problem.severity error - * @precision high + * @precision medium * @tags security * external/cwe/cwe-022 */ diff --git a/javascript/ql/src/Security/CWE-022/ZipSlipGood.js b/javascript/ql/src/Security/CWE-022/ZipSlipGood.js new file mode 100644 index 000000000000..325e9586223f --- /dev/null +++ b/javascript/ql/src/Security/CWE-022/ZipSlipGood.js @@ -0,0 +1,14 @@ +const fs = require('fs'); +const unzip = require('unzip'); + +fs.createReadStream('archive.zip') + .pipe(unzip.Parse()) + .on('entry', entry => { + const fileName = entry.path; + if (entry.path.indexOf('..') == -1) { + entry.pipe(fs.createWriteStream(entry.path)); + } + else { + console.log('skipping bad path', entry.path); + } + }); diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll index f21c02e04c5e..0d000b132609 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll @@ -20,7 +20,7 @@ module ZipSlip { */ abstract class SanitizerGuard extends TaintTracking::SanitizerGuardNode, DataFlow::ValueNode { } - /** A taint tracking configuration for Zip Slip */ + /** A taint tracking configuration for unsafe zip extraction. */ class Configuration extends TaintTracking::Configuration { Configuration() { this = "ZipSlip" } @@ -36,51 +36,54 @@ module ZipSlip { /** * Gets a node that can be a parsed zip archive. */ - DataFlow::SourceNode parsedArchive() { + private DataFlow::SourceNode parsedArchive() { result = DataFlow::moduleImport("unzip").getAMemberCall("Parse") or // `streamProducer.pipe(unzip.Parse())` is a typical (but not // universal) pattern when using nodejs streams, whose return // value is the parsed stream. - exists(DataFlow::MethodCallNode pipe | pipe.getMethodName() = "pipe" - and parsedArchive().flowsTo(pipe.getArgument(0))) + exists(DataFlow::MethodCallNode pipe | + pipe.getMethodName() = "pipe" and + parsedArchive().flowsTo(pipe.getArgument(0)) + ) } - /** - * An access to the filepath of an entry of a zipfile being extracted - * by npm module `unzip`. For example, in - * ```javascript - * const unzip = require('unzip'); - * - * fs.createReadStream('archive.zip') - * .pipe(unzip.Parse()) - * .on('entry', entry => { - * const path = entry.path; - * }); - * ``` - * there is an `UnzipEntrySource` node corresponding to - * the expression `entry.path`. - */ + /** A zip archive entry path access, as a source for unsafe zip extraction. */ class UnzipEntrySource extends Source { + // For example, in + // ```javascript + // const unzip = require('unzip'); + // + // fs.createReadStream('archive.zip') + // .pipe(unzip.Parse()) + // .on('entry', entry => { + // const path = entry.path; + // }); + // ``` + // there is an `UnzipEntrySource` node corresponding to + // the expression `entry.path`. UnzipEntrySource() { - this = parsedArchive().getAMemberCall("on").getCallback(1).getParameter(0).getAPropertyRead("path") + this = parsedArchive() + .getAMemberCall("on") + .getCallback(1) + .getParameter(0) + .getAPropertyRead("path") } } - /** - * A sink that is the path that a createWriteStream gets created at. - * This is not covered by FileSystemWriteSink, because it is - * required that a write actually takes place to the stream. - * However, we want to consider even the bare createWriteStream to - * be a zipslip vulnerability since it may truncate an existing file. - */ + /** A call to `fs.createWriteStream`, as a sink for unsafe zip extraction. */ class CreateWriteStreamSink extends Sink { CreateWriteStreamSink() { + // This is not covered by `FileSystemWriteSink`, because it is + // required that a write actually takes place to the stream. + // However, we want to consider even the bare createWriteStream + // to be a zipslip vulnerability since it may truncate an + // existing file. this = DataFlow::moduleImport("fs").getAMemberCall("createWriteStream").getArgument(0) } } - /** A sink that is a file path that gets written to. */ + /** A file path of a file write, as a sink for unsafe zip extraction. */ class FileSystemWriteSink extends Sink { FileSystemWriteSink() { exists(FileSystemWriteAccess fsw | fsw.getAPathArgument() = this) } } @@ -89,7 +92,7 @@ module ZipSlip { * Gets a string which suffices to search for to ensure that a * filepath will not refer to parent directories. */ - string getAParentDirName() { result = ".." or result = "../" } + private string getAParentDirName() { result = ".." or result = "../" } /** A check that a path string does not include '..' */ class NoParentDirSanitizerGuard extends SanitizerGuard { From caebdd2f688b270f0104487d7baf7dd5d02e0f93 Mon Sep 17 00:00:00 2001 From: Jason Reed Date: Wed, 27 Feb 2019 09:16:45 -0500 Subject: [PATCH 62/71] JS: Fix incorrect sample link --- javascript/ql/src/Security/CWE-022/ZipSlip.qhelp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp b/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp index 07c4fb0c5790..536347701133 100644 --- a/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp +++ b/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp @@ -48,7 +48,7 @@ instance, if it were created by something like zip archive.zip contain any ".." in it.

      - + From 674d2790b4c0077b7c30f508a23919c9bc1c715c Mon Sep 17 00:00:00 2001 From: Jason Reed Date: Wed, 27 Feb 2019 10:22:06 -0500 Subject: [PATCH 63/71] JS: Address review comments --- javascript/ql/src/Security/CWE-022/ZipSlip.qhelp | 2 +- .../semmle/javascript/security/dataflow/ZipSlip.qll | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp b/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp index 536347701133..d708965a8818 100644 --- a/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp +++ b/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp @@ -45,7 +45,7 @@ instance, if it were created by something like zip archive.zip

      To fix this vulnerability, we can to check that the path does not -contain any ".." in it. +contain any ".." elements in it.

      diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll index 0d000b132609..a752b25d59c0 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll @@ -43,6 +43,7 @@ module ZipSlip { // universal) pattern when using nodejs streams, whose return // value is the parsed stream. exists(DataFlow::MethodCallNode pipe | + pipe = result and pipe.getMethodName() = "pipe" and parsedArchive().flowsTo(pipe.getArgument(0)) ) @@ -63,11 +64,12 @@ module ZipSlip { // there is an `UnzipEntrySource` node corresponding to // the expression `entry.path`. UnzipEntrySource() { - this = parsedArchive() - .getAMemberCall("on") - .getCallback(1) - .getParameter(0) - .getAPropertyRead("path") + exists(DataFlow::CallNode cn | + cn = parsedArchive().getAMemberCall("on") and + cn.getArgument(0).mayHaveStringValue("entry") and + this = cn.getCallback(1) + .getParameter(0) + .getAPropertyRead("path")) } } From c5e57dacf88ff31175ad334cf77742a035c0801d Mon Sep 17 00:00:00 2001 From: Jason Reed Date: Wed, 27 Feb 2019 10:23:32 -0500 Subject: [PATCH 64/71] JS: Actually use fileName in examples --- javascript/ql/src/Security/CWE-022/ZipSlipBad.js | 2 +- javascript/ql/src/Security/CWE-022/ZipSlipGood.js | 6 +++--- .../query-tests/Security/CWE-022/ZipSlip/ZipSlip.expected | 8 ++++++-- .../query-tests/Security/CWE-022/ZipSlip/ZipSlipBad.js | 2 +- .../query-tests/Security/CWE-022/ZipSlip/ZipSlipGood.js | 6 +++--- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/javascript/ql/src/Security/CWE-022/ZipSlipBad.js b/javascript/ql/src/Security/CWE-022/ZipSlipBad.js index fc6ac2514485..e4fdbe7d1f38 100644 --- a/javascript/ql/src/Security/CWE-022/ZipSlipBad.js +++ b/javascript/ql/src/Security/CWE-022/ZipSlipBad.js @@ -5,5 +5,5 @@ fs.createReadStream('archive.zip') .pipe(unzip.Parse()) .on('entry', entry => { const fileName = entry.path; - entry.pipe(fs.createWriteStream(entry.path)); + entry.pipe(fs.createWriteStream(fileName)); }); diff --git a/javascript/ql/src/Security/CWE-022/ZipSlipGood.js b/javascript/ql/src/Security/CWE-022/ZipSlipGood.js index 325e9586223f..6a55fc81715d 100644 --- a/javascript/ql/src/Security/CWE-022/ZipSlipGood.js +++ b/javascript/ql/src/Security/CWE-022/ZipSlipGood.js @@ -5,10 +5,10 @@ fs.createReadStream('archive.zip') .pipe(unzip.Parse()) .on('entry', entry => { const fileName = entry.path; - if (entry.path.indexOf('..') == -1) { - entry.pipe(fs.createWriteStream(entry.path)); + if (fileName.indexOf('..') == -1) { + entry.pipe(fs.createWriteStream(fileName)); } else { - console.log('skipping bad path', entry.path); + console.log('skipping bad path', fileName); } }); diff --git a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.expected b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.expected index a9e316d7d469..5dae853958e9 100644 --- a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.expected +++ b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlip.expected @@ -3,11 +3,15 @@ nodes | ZipSlipBad2.js:5:20:5:46 | 'output ... ry.path | | ZipSlipBad2.js:5:37:5:46 | entry.path | | ZipSlipBad2.js:6:22:6:29 | fileName | -| ZipSlipBad.js:8:37:8:46 | entry.path | +| ZipSlipBad.js:7:11:7:31 | fileName | +| ZipSlipBad.js:7:22:7:31 | entry.path | +| ZipSlipBad.js:8:37:8:44 | fileName | edges | ZipSlipBad2.js:5:9:5:46 | fileName | ZipSlipBad2.js:6:22:6:29 | fileName | | ZipSlipBad2.js:5:20:5:46 | 'output ... ry.path | ZipSlipBad2.js:5:9:5:46 | fileName | | ZipSlipBad2.js:5:37:5:46 | entry.path | ZipSlipBad2.js:5:20:5:46 | 'output ... ry.path | +| ZipSlipBad.js:7:11:7:31 | fileName | ZipSlipBad.js:8:37:8:44 | fileName | +| ZipSlipBad.js:7:22:7:31 | entry.path | ZipSlipBad.js:7:11:7:31 | fileName | #select | ZipSlipBad2.js:6:22:6:29 | fileName | ZipSlipBad2.js:5:37:5:46 | entry.path | ZipSlipBad2.js:6:22:6:29 | fileName | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlipBad2.js:5:37:5:46 | entry.path | item path | -| ZipSlipBad.js:8:37:8:46 | entry.path | ZipSlipBad.js:8:37:8:46 | entry.path | ZipSlipBad.js:8:37:8:46 | entry.path | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlipBad.js:8:37:8:46 | entry.path | item path | +| ZipSlipBad.js:8:37:8:44 | fileName | ZipSlipBad.js:7:22:7:31 | entry.path | ZipSlipBad.js:8:37:8:44 | fileName | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlipBad.js:7:22:7:31 | entry.path | item path | diff --git a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipBad.js b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipBad.js index fc6ac2514485..e4fdbe7d1f38 100644 --- a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipBad.js +++ b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipBad.js @@ -5,5 +5,5 @@ fs.createReadStream('archive.zip') .pipe(unzip.Parse()) .on('entry', entry => { const fileName = entry.path; - entry.pipe(fs.createWriteStream(entry.path)); + entry.pipe(fs.createWriteStream(fileName)); }); diff --git a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipGood.js b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipGood.js index 325e9586223f..6a55fc81715d 100644 --- a/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipGood.js +++ b/javascript/ql/test/query-tests/Security/CWE-022/ZipSlip/ZipSlipGood.js @@ -5,10 +5,10 @@ fs.createReadStream('archive.zip') .pipe(unzip.Parse()) .on('entry', entry => { const fileName = entry.path; - if (entry.path.indexOf('..') == -1) { - entry.pipe(fs.createWriteStream(entry.path)); + if (fileName.indexOf('..') == -1) { + entry.pipe(fs.createWriteStream(fileName)); } else { - console.log('skipping bad path', entry.path); + console.log('skipping bad path', fileName); } }); From c1b218a5ffe4721e64f283a6d96b09ed45393d20 Mon Sep 17 00:00:00 2001 From: Jason Reed Date: Thu, 28 Feb 2019 06:25:25 -0500 Subject: [PATCH 65/71] JS: Documentation fixes --- .../ql/src/Security/CWE-022/ZipSlip.qhelp | 21 +++++++++++++++---- .../ql/src/Security/CWE-022/ZipSlipBad.js | 1 + .../ql/src/Security/CWE-022/ZipSlipGood.js | 1 + .../javascript/security/dataflow/ZipSlip.qll | 6 +++--- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp b/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp index d708965a8818..7a3619472166 100644 --- a/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp +++ b/javascript/ql/src/Security/CWE-022/ZipSlip.qhelp @@ -36,19 +36,32 @@ to prevent writing files to unexpected locations.

      -Here is an example of extracting an archive without validating -filenames. If archive.zip contained relative paths (for +In this example an archive is extracted without validating file paths. +If archive.zip contained relative paths (for instance, if it were created by something like zip archive.zip -../file.txt) then executing this code would write to those paths. +../file.txt) then executing this code could write to locations +outside the destination directory.

      -

      To fix this vulnerability, we can to check that the path does not +

      To fix this vulnerability, we need to check that the path does not contain any ".." elements in it.

      + + +
    • +Snyk: +Zip Slip Vulnerability. +
      • +
    • +OWASP: +Path Traversal. +
      • + +       diff --git a/javascript/ql/src/Security/CWE-022/ZipSlipBad.js b/javascript/ql/src/Security/CWE-022/ZipSlipBad.js index e4fdbe7d1f38..0b993cbfec35 100644 --- a/javascript/ql/src/Security/CWE-022/ZipSlipBad.js +++ b/javascript/ql/src/Security/CWE-022/ZipSlipBad.js @@ -5,5 +5,6 @@ fs.createReadStream('archive.zip') .pipe(unzip.Parse()) .on('entry', entry => { const fileName = entry.path; + // BAD: This could write any file on the filesystem. entry.pipe(fs.createWriteStream(fileName)); }); diff --git a/javascript/ql/src/Security/CWE-022/ZipSlipGood.js b/javascript/ql/src/Security/CWE-022/ZipSlipGood.js index 6a55fc81715d..0d72a5537af0 100644 --- a/javascript/ql/src/Security/CWE-022/ZipSlipGood.js +++ b/javascript/ql/src/Security/CWE-022/ZipSlipGood.js @@ -5,6 +5,7 @@ fs.createReadStream('archive.zip') .pipe(unzip.Parse()) .on('entry', entry => { const fileName = entry.path; + // GOOD: ensures the path is safe to write to. if (fileName.indexOf('..') == -1) { entry.pipe(fs.createWriteStream(fileName)); } diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll index a752b25d59c0..2182fc038878 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/ZipSlip.qll @@ -78,7 +78,7 @@ module ZipSlip { CreateWriteStreamSink() { // This is not covered by `FileSystemWriteSink`, because it is // required that a write actually takes place to the stream. - // However, we want to consider even the bare createWriteStream + // However, we want to consider even the bare `createWriteStream` // to be a zipslip vulnerability since it may truncate an // existing file. this = DataFlow::moduleImport("fs").getAMemberCall("createWriteStream").getArgument(0) @@ -91,8 +91,8 @@ module ZipSlip { } /** - * Gets a string which suffices to search for to ensure that a - * filepath will not refer to parent directories. + * Gets a string which is sufficient to exclude to make + * a filepath definitely not refer to parent directories. */ private string getAParentDirName() { result = ".." or result = "../" } From 86bbb5fb18f3b59d2af19848c0c924ee67f2d0cc Mon Sep 17 00:00:00 2001 From: Jason Reed Date: Thu, 28 Feb 2019 09:34:21 -0500 Subject: [PATCH 66/71] JS: Add ZipSlip query to security suite --- javascript/config/suites/javascript/security | 1 + 1 file changed, 1 insertion(+) diff --git a/javascript/config/suites/javascript/security b/javascript/config/suites/javascript/security index f572def7e8eb..c1934ae1d5f6 100644 --- a/javascript/config/suites/javascript/security +++ b/javascript/config/suites/javascript/security @@ -4,6 +4,7 @@ + semmlecode-javascript-queries/Security/CWE-020/IncompleteUrlSubstringSanitization.ql: /Security/CWE/CWE-020 + semmlecode-javascript-queries/Security/CWE-020/IncorrectSuffixCheck.ql: /Security/CWE/CWE-020 + semmlecode-javascript-queries/Security/CWE-022/TaintedPath.ql: /Security/CWE/CWE-022 ++ semmlecode-javascript-queries/Security/CWE-022/ZipSlip.ql: /Security/CWE/CWE-022 + semmlecode-javascript-queries/Security/CWE-078/CommandInjection.ql: /Security/CWE/CWE-078 + semmlecode-javascript-queries/Security/CWE-079/ReflectedXss.ql: /Security/CWE/CWE-079 + semmlecode-javascript-queries/Security/CWE-079/StoredXss.ql: /Security/CWE/CWE-079 From 91cfc9bd4c547d90adfd01e594a8073fe7b5a1d4 Mon Sep 17 00:00:00 2001 From: Taus Brock-Nannestad Date: Fri, 1 Mar 2019 11:06:48 +0100 Subject: [PATCH 67/71] Change kind to `path-problem`. --- python/ql/src/Functions/ModificationOfParameterWithDefault.ql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ql/src/Functions/ModificationOfParameterWithDefault.ql b/python/ql/src/Functions/ModificationOfParameterWithDefault.ql index 2610870ee341..206174e02b6a 100644 --- a/python/ql/src/Functions/ModificationOfParameterWithDefault.ql +++ b/python/ql/src/Functions/ModificationOfParameterWithDefault.ql @@ -2,7 +2,7 @@ * @name Modification of parameter with default * @description Modifying the default value of a parameter can lead to unexpected * results. - * @kind problem + * @kind path-problem * @tags reliability * maintainability * @problem.severity error From ebd9bc3cb580bcfd48027637b04e8c1d196fb180 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 1 Mar 2019 10:54:05 +0000 Subject: [PATCH 68/71] Python: Improve taint tracking to account for truthiness of the taint kind. --- .../src/semmle/python/security/TaintTracking.qll | 15 ++++++++++++++- .../library-tests/taint/general/TestNode.expected | 5 +++++ .../library-tests/taint/general/TestSink.expected | 2 ++ .../taint/general/TestSource.expected | 1 + .../library-tests/taint/general/TestStep.expected | 4 ++++ .../library-tests/taint/general/TestVar.expected | 5 +++++ .../ql/test/library-tests/taint/general/test.py | 11 +++++++++++ 7 files changed, 42 insertions(+), 1 deletion(-) diff --git a/python/ql/src/semmle/python/security/TaintTracking.qll b/python/ql/src/semmle/python/security/TaintTracking.qll index bd0037b52caf..16cd1952a90d 100755 --- a/python/ql/src/semmle/python/security/TaintTracking.qll +++ b/python/ql/src/semmle/python/security/TaintTracking.qll @@ -148,6 +148,16 @@ abstract class TaintKind extends string { none() } + /** Gets the boolean values (may be one, neither, or both) that + * may result from the Python expression `bool(this)` + */ + boolean booleanValue() { + /* Default to true as the vast majority of taint is strings and + * the empty string is almost always benign. + */ + result = true + } + string repr() { result = this } } @@ -1190,7 +1200,8 @@ library module TaintFlowImplementation { sanitizer.sanitizingEdge(kind, test) ) | - not Filters::isinstance(test.getTest(), _, var.getSourceVariable().getAUse()) + not Filters::isinstance(test.getTest(), _, var.getSourceVariable().getAUse()) and + not test.getTest() = var.getSourceVariable().getAUse() or exists(ControlFlowNode c, ClassObject cls | Filters::isinstance(test.getTest(), c, var.getSourceVariable().getAUse()) @@ -1200,6 +1211,8 @@ library module TaintFlowImplementation { or test.getSense() = false and not kind.getClass().getAnImproperSuperType() = cls ) + or + test.getTest() = var.getSourceVariable().getAUse() and kind.booleanValue() = test.getSense() ) } diff --git a/python/ql/test/library-tests/taint/general/TestNode.expected b/python/ql/test/library-tests/taint/general/TestNode.expected index c7ae50289a07..b1f290b4acba 100644 --- a/python/ql/test/library-tests/taint/general/TestNode.expected +++ b/python/ql/test/library-tests/taint/general/TestNode.expected @@ -215,6 +215,11 @@ | Taint simple.test | test.py:169 | SOURCE | | | Taint simple.test | test.py:172 | Subscript | | | Taint simple.test | test.py:173 | Subscript | | +| Taint simple.test | test.py:178 | SOURCE | | +| Taint simple.test | test.py:179 | t | | +| Taint simple.test | test.py:180 | t | | +| Taint simple.test | test.py:183 | t | | +| Taint simple.test | test.py:186 | t | | | Taint {simple.test} | test.py:169 | Dict | | | Taint {simple.test} | test.py:171 | d | | | Taint {simple.test} | test.py:173 | y | | diff --git a/python/ql/test/library-tests/taint/general/TestSink.expected b/python/ql/test/library-tests/taint/general/TestSink.expected index dd169d78f033..c8f7f22a1fa4 100644 --- a/python/ql/test/library-tests/taint/general/TestSink.expected +++ b/python/ql/test/library-tests/taint/general/TestSink.expected @@ -32,3 +32,5 @@ | simple.test | test.py:159 | 160 | t | simple.test | | simple.test | test.py:168 | 172 | Subscript | simple.test | | simple.test | test.py:169 | 173 | Subscript | simple.test | +| simple.test | test.py:178 | 180 | t | simple.test | +| simple.test | test.py:178 | 186 | t | simple.test | diff --git a/python/ql/test/library-tests/taint/general/TestSource.expected b/python/ql/test/library-tests/taint/general/TestSource.expected index ea9012b7735b..d341758b2eae 100644 --- a/python/ql/test/library-tests/taint/general/TestSource.expected +++ b/python/ql/test/library-tests/taint/general/TestSource.expected @@ -40,3 +40,4 @@ | test.py:163 | SOURCE | simple.test | | test.py:168 | SOURCE | simple.test | | test.py:169 | SOURCE | simple.test | +| test.py:178 | SOURCE | simple.test | diff --git a/python/ql/test/library-tests/taint/general/TestStep.expected b/python/ql/test/library-tests/taint/general/TestStep.expected index a4c72fd6b0ce..dec3e5b03b24 100644 --- a/python/ql/test/library-tests/taint/general/TestStep.expected +++ b/python/ql/test/library-tests/taint/general/TestStep.expected @@ -173,6 +173,10 @@ | Taint simple.test | test.py:163 | SOURCE | | --> | Taint simple.test | test.py:164 | s | | | Taint simple.test | test.py:168 | SOURCE | | --> | Taint [simple.test] | test.py:168 | List | | | Taint simple.test | test.py:169 | SOURCE | | --> | Taint {simple.test} | test.py:169 | Dict | | +| Taint simple.test | test.py:178 | SOURCE | | --> | Taint simple.test | test.py:179 | t | | +| Taint simple.test | test.py:178 | SOURCE | | --> | Taint simple.test | test.py:180 | t | | +| Taint simple.test | test.py:178 | SOURCE | | --> | Taint simple.test | test.py:183 | t | | +| Taint simple.test | test.py:178 | SOURCE | | --> | Taint simple.test | test.py:186 | t | | | Taint {simple.test} | test.py:169 | Dict | | --> | Taint {simple.test} | test.py:171 | d | | | Taint {simple.test} | test.py:169 | Dict | | --> | Taint {simple.test} | test.py:175 | d | | | Taint {simple.test} | test.py:171 | d | | --> | Taint {simple.test} | test.py:173 | y | | diff --git a/python/ql/test/library-tests/taint/general/TestVar.expected b/python/ql/test/library-tests/taint/general/TestVar.expected index 5aad11fb5476..9939820a0a4d 100644 --- a/python/ql/test/library-tests/taint/general/TestVar.expected +++ b/python/ql/test/library-tests/taint/general/TestVar.expected @@ -177,3 +177,8 @@ | test.py:174 | l_2 | test.py:168 | Taint [simple.test] | List | | test.py:175 | d2_0 | test.py:175 | Taint {simple.test} | dict() | | test.py:175 | d_2 | test.py:169 | Taint {simple.test} | Dict | +| test.py:178 | t_0 | test.py:178 | Taint simple.test | SOURCE | +| test.py:180 | t_1 | test.py:178 | Taint simple.test | SOURCE | +| test.py:180 | t_2 | test.py:178 | Taint simple.test | SOURCE | +| test.py:183 | t_3 | test.py:178 | Taint simple.test | SOURCE | +| test.py:186 | t_4 | test.py:178 | Taint simple.test | SOURCE | diff --git a/python/ql/test/library-tests/taint/general/test.py b/python/ql/test/library-tests/taint/general/test.py index 5a6fc1e2900d..6c752b2c1d71 100644 --- a/python/ql/test/library-tests/taint/general/test.py +++ b/python/ql/test/library-tests/taint/general/test.py @@ -173,3 +173,14 @@ def test_update_extend(x, y): SINK(y["key"]) l2 = list(l) d2 = dict(d) + +def test_truth(): + t = SOURCE + if t: + SINK(t) + else: + SINK(t) + if not t: + SINK(t) + else: + SINK(t) From 94190e76aae2b604e21d757bbd449f7661b5fa98 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 1 Mar 2019 12:01:39 +0000 Subject: [PATCH 69/71] Python: Update py/modification-of-default-value to account for truthiness of default value. --- .../ModificationOfParameterWithDefault.ql | 57 ++++++++++++------- ...odificationOfParameterWithDefault.expected | 36 ++++++------ .../Functions/general/functions_test.py | 6 ++ 3 files changed, 63 insertions(+), 36 deletions(-) diff --git a/python/ql/src/Functions/ModificationOfParameterWithDefault.ql b/python/ql/src/Functions/ModificationOfParameterWithDefault.ql index 206174e02b6a..ab94a9735763 100644 --- a/python/ql/src/Functions/ModificationOfParameterWithDefault.ql +++ b/python/ql/src/Functions/ModificationOfParameterWithDefault.ql @@ -19,32 +19,47 @@ predicate safe_method(string name) { name = "items" or name = "keys" or name = "values" or name = "iteritems" or name = "iterkeys" or name = "itervalues" } -predicate maybe_parameter(SsaVariable var, Function f, Parameter p) { - p = var.getAnUltimateDefinition().getDefinition().getNode() and - f.getAnArg() = p +/** Gets the truthiness (non emptyness) of the default of `p` if that value is mutable */ +private boolean mutableDefaultValue(Parameter p) { + exists(Dict d | + p.getDefault() = d | + exists(d.getAKey()) and result = true + or + not exists(d.getAKey()) and result = false + ) + or + exists(List l | + p.getDefault() = l | + exists(l.getAnElt()) and result = true + or + not exists(l.getAnElt()) and result = false + ) } -predicate has_mutable_default(Parameter p) { - exists(SsaVariable v, FunctionExpr f | maybe_parameter(v, f.getInnerScope(), p) and - exists(int i, int def_cnt, int arg_cnt | - def_cnt = count(f.getArgs().getADefault()) and - arg_cnt = count(f.getInnerScope().getAnArg()) and - i in [1 .. arg_cnt] and - (f.getArgs().getDefault(def_cnt - i) instanceof Dict or f.getArgs().getDefault(def_cnt - i) instanceof List) and - f.getInnerScope().getArgName(arg_cnt - i) = v.getId() - ) - ) + +class NonEmptyMutableValue extends TaintKind { + NonEmptyMutableValue() { + this = "non-empty mutable value" + } } -class MutableValue extends TaintKind { - MutableValue() { - this = "mutable value" +class EmptyMutableValue extends TaintKind { + EmptyMutableValue() { + this = "empty mutable value" } + + override boolean booleanValue() { + result = false + } + } class MutableDefaultValue extends TaintSource { + + boolean nonEmpty; + MutableDefaultValue() { - has_mutable_default(this.(NameNode).getNode()) + nonEmpty = mutableDefaultValue(this.(NameNode).getNode()) } override string toString() { @@ -52,7 +67,9 @@ class MutableDefaultValue extends TaintSource { } override predicate isSourceOf(TaintKind kind) { - kind instanceof MutableValue + nonEmpty = false and kind instanceof EmptyMutableValue + or + nonEmpty = true and kind instanceof NonEmptyMutableValue } } @@ -68,7 +85,9 @@ class Mutation extends TaintSink { } override predicate sinks(TaintKind kind) { - kind instanceof MutableValue + kind instanceof EmptyMutableValue + or + kind instanceof NonEmptyMutableValue } } diff --git a/python/ql/test/query-tests/Functions/general/ModificationOfParameterWithDefault.expected b/python/ql/test/query-tests/Functions/general/ModificationOfParameterWithDefault.expected index fa8feaf186ba..a65d2ca8dc15 100644 --- a/python/ql/test/query-tests/Functions/general/ModificationOfParameterWithDefault.expected +++ b/python/ql/test/query-tests/Functions/general/ModificationOfParameterWithDefault.expected @@ -1,20 +1,22 @@ edges -| functions_test.py:36:9:36:9 | mutable value | functions_test.py:37:16:37:16 | mutable value | -| functions_test.py:39:9:39:9 | mutable value | functions_test.py:40:5:40:5 | mutable value | -| functions_test.py:238:15:238:15 | mutable value | functions_test.py:239:5:239:5 | mutable value | -| functions_test.py:290:25:290:25 | mutable value | functions_test.py:291:5:291:5 | mutable value | -| functions_test.py:293:21:293:21 | mutable value | functions_test.py:294:5:294:5 | mutable value | -| functions_test.py:296:27:296:27 | mutable value | functions_test.py:297:25:297:25 | mutable value | -| functions_test.py:296:27:296:27 | mutable value | functions_test.py:298:21:298:21 | mutable value | -| functions_test.py:297:25:297:25 | mutable value | functions_test.py:290:25:290:25 | mutable value | -| functions_test.py:298:21:298:21 | mutable value | functions_test.py:293:21:293:21 | mutable value | +| functions_test.py:36:9:36:9 | empty mutable value | functions_test.py:37:16:37:16 | empty mutable value | +| functions_test.py:39:9:39:9 | empty mutable value | functions_test.py:40:5:40:5 | empty mutable value | +| functions_test.py:238:15:238:15 | empty mutable value | functions_test.py:239:5:239:5 | empty mutable value | +| functions_test.py:290:25:290:25 | empty mutable value | functions_test.py:291:5:291:5 | empty mutable value | +| functions_test.py:293:21:293:21 | empty mutable value | functions_test.py:294:5:294:5 | empty mutable value | +| functions_test.py:296:27:296:27 | empty mutable value | functions_test.py:297:25:297:25 | empty mutable value | +| functions_test.py:296:27:296:27 | empty mutable value | functions_test.py:298:21:298:21 | empty mutable value | +| functions_test.py:297:25:297:25 | empty mutable value | functions_test.py:290:25:290:25 | empty mutable value | +| functions_test.py:298:21:298:21 | empty mutable value | functions_test.py:293:21:293:21 | empty mutable value | +| functions_test.py:300:26:300:26 | empty mutable value | functions_test.py:301:8:301:8 | empty mutable value | +| functions_test.py:300:26:300:26 | empty mutable value | functions_test.py:303:12:303:12 | empty mutable value | parents -| functions_test.py:290:25:290:25 | mutable value | functions_test.py:297:25:297:25 | mutable value | -| functions_test.py:291:5:291:5 | mutable value | functions_test.py:297:25:297:25 | mutable value | -| functions_test.py:293:21:293:21 | mutable value | functions_test.py:298:21:298:21 | mutable value | -| functions_test.py:294:5:294:5 | mutable value | functions_test.py:298:21:298:21 | mutable value | +| functions_test.py:290:25:290:25 | empty mutable value | functions_test.py:297:25:297:25 | empty mutable value | +| functions_test.py:291:5:291:5 | empty mutable value | functions_test.py:297:25:297:25 | empty mutable value | +| functions_test.py:293:21:293:21 | empty mutable value | functions_test.py:298:21:298:21 | empty mutable value | +| functions_test.py:294:5:294:5 | empty mutable value | functions_test.py:298:21:298:21 | empty mutable value | #select -| functions_test.py:40:5:40:5 | Taint sink | functions_test.py:39:9:39:9 | mutable value | functions_test.py:40:5:40:5 | mutable value | $@ flows to here and is mutated. | functions_test.py:39:9:39:9 | mutable default value | Default value | -| functions_test.py:239:5:239:5 | Taint sink | functions_test.py:238:15:238:15 | mutable value | functions_test.py:239:5:239:5 | mutable value | $@ flows to here and is mutated. | functions_test.py:238:15:238:15 | mutable default value | Default value | -| functions_test.py:291:5:291:5 | Taint sink | functions_test.py:296:27:296:27 | mutable value | functions_test.py:291:5:291:5 | mutable value | $@ flows to here and is mutated. | functions_test.py:296:27:296:27 | mutable default value | Default value | -| functions_test.py:294:5:294:5 | Taint sink | functions_test.py:296:27:296:27 | mutable value | functions_test.py:294:5:294:5 | mutable value | $@ flows to here and is mutated. | functions_test.py:296:27:296:27 | mutable default value | Default value | +| functions_test.py:40:5:40:5 | Taint sink | functions_test.py:39:9:39:9 | empty mutable value | functions_test.py:40:5:40:5 | empty mutable value | $@ flows to here and is mutated. | functions_test.py:39:9:39:9 | mutable default value | Default value | +| functions_test.py:239:5:239:5 | Taint sink | functions_test.py:238:15:238:15 | empty mutable value | functions_test.py:239:5:239:5 | empty mutable value | $@ flows to here and is mutated. | functions_test.py:238:15:238:15 | mutable default value | Default value | +| functions_test.py:291:5:291:5 | Taint sink | functions_test.py:296:27:296:27 | empty mutable value | functions_test.py:291:5:291:5 | empty mutable value | $@ flows to here and is mutated. | functions_test.py:296:27:296:27 | mutable default value | Default value | +| functions_test.py:294:5:294:5 | Taint sink | functions_test.py:296:27:296:27 | empty mutable value | functions_test.py:294:5:294:5 | empty mutable value | $@ flows to here and is mutated. | functions_test.py:296:27:296:27 | mutable default value | Default value | diff --git a/python/ql/test/query-tests/Functions/general/functions_test.py b/python/ql/test/query-tests/Functions/general/functions_test.py index 280af2bc2279..91613675aec1 100644 --- a/python/ql/test/query-tests/Functions/general/functions_test.py +++ b/python/ql/test/query-tests/Functions/general/functions_test.py @@ -296,3 +296,9 @@ def mutate_argument(x): def indirect_modification(y = []): aug_assign_argument(y) mutate_argument(y) + +def guarded_modification(z=[]): + if z: + z.append(0) + return z + From af397d3546c34c2337a843a840b794ad791750b5 Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Fri, 1 Mar 2019 13:30:05 +0000 Subject: [PATCH 70/71] Changenotes: Fix copy/paste-o. --- change-notes/1.20/analysis-cpp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/change-notes/1.20/analysis-cpp.md b/change-notes/1.20/analysis-cpp.md index 46a0929850e6..3021fe1b610e 100644 --- a/change-notes/1.20/analysis-cpp.md +++ b/change-notes/1.20/analysis-cpp.md @@ -37,4 +37,4 @@ * There is a new `Namespace.isInline()` predicate, which holds if the namespace was declared as `inline namespace`. * The `Expr.isConstant()` predicate now also holds for _address constant expressions_, which are addresses that will be constant after the program has been linked. These address constants do not have a result for `Expr.getValue()`. * There are new `Function.isDeclaredConstexpr()` and `Function.isConstexpr()` predicates. They can be used to tell whether a function was declared as `constexpr`, and whether it actually is `constexpr`. -* There is a new `Variable.isConstexpr()` predicates. It can be used to tell whether a variable is `constexpr`. +* There is a new `Variable.isConstexpr()` predicate. It can be used to tell whether a variable is `constexpr`. From 4f9ffb38e6de89d2181457e4d8bfbf73073c4391 Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Mon, 4 Mar 2019 09:47:38 +0100 Subject: [PATCH 71/71] C++: Set cpp/command-line-injection precision=low This query is only appropriate for setuid programs. Since such programs are at most 0.1% of all code we analyse, I would say this query has a precision of at most 0.1%. --- cpp/ql/src/Security/CWE/CWE-078/ExecTainted.ql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/ql/src/Security/CWE/CWE-078/ExecTainted.ql b/cpp/ql/src/Security/CWE/CWE-078/ExecTainted.ql index 17d5a62b2716..894212cd4cf9 100644 --- a/cpp/ql/src/Security/CWE/CWE-078/ExecTainted.ql +++ b/cpp/ql/src/Security/CWE/CWE-078/ExecTainted.ql @@ -5,7 +5,7 @@ * to command injection. * @kind problem * @problem.severity error - * @precision high + * @precision low * @id cpp/command-line-injection * @tags security * external/cwe/cwe-078