Python: New points-to and object model with performance fixes

python/ql/src/Exceptions/UnguardedNextInGenerator.ql

@@ -16,7 +16,7 @@ FunctionObject iter() {
     result = Object::builtin("iter")
 }
 
-FunctionObject next() {
+BuiltinFunctionObject next() {
     result = Object::builtin("next")
 }
 

python/ql/src/Expressions/IsComparisons.qll

@@ -104,8 +104,14 @@ predicate invalid_portable_is_comparison(Compare comp, Cmpop op, ClassObject cls
     /* OK to use 'is' when comparing items from a known set of objects */
     not exists(Expr left, Expr right, Object obj |
         comp.compares(left, op, right) and
-        left.refersTo(obj) and right.refersTo(obj) and
-        exists(ImmutableLiteral il | il.getLiteralObject() = obj)
+        exists(ImmutableLiteral il | il.getLiteralObject() = obj) |
+        left.refersTo(obj) and right.refersTo(obj)
+        or
+        /* Simple constant in module, probably some sort of sentinel */
+        exists(AstNode origin |
+            not left.refersTo(_) and right.refersTo(obj, origin) and
+            origin.getScope().getEnclosingModule() = comp.getScope().getEnclosingModule()
+        )
     )
     and
     /* OK to use 'is' when comparing with a member of an enum */

python/ql/src/Security/CWE-798/HardcodedCredentials.ql

@@ -53,6 +53,10 @@ predicate capitalized_word(StrConst str) {
     str.getText().regexpMatch("[A-Z][a-z]+")
 }
 
+predicate format_string(StrConst str) {
+    str.getText().matches("%{%}%")
+}
+
 predicate maybeCredential(ControlFlowNode f) {
     /* A string that is not too short and unlikely to be text or an identifier. */
     exists(StrConst str |
@@ -66,20 +70,21 @@ predicate maybeCredential(ControlFlowNode f) {
         /* Not too repetitive */
         exists(int chars |
             chars = char_count(str) |
-            chars > 20 or
-            chars > str.getText().length()/2
+            chars > 15 or
+            chars*3 > str.getText().length()*2
         ) and
         not possible_reflective_name(str.getText()) and
-        not capitalized_word(str)
+        not capitalized_word(str) and
+        not format_string(str)
     )
     or
-    /* Or, an integer with at least 8 digits */
+    /* Or, an integer with over 32 bits */
     exists(IntegerLiteral lit |
         f.getNode() = lit
         |
-        not exists(lit.getValue())
-        or
-        lit.getValue() > 10000000
+        not exists(lit.getValue()) and
+        /* Not a set of flags or round number */
+        not lit.getN().matches("%00%")
     )
 }
 

python/ql/src/analysis/CallGraphEfficiency.ql

@@ -9,15 +9,15 @@ import semmle.python.pointsto.PointsToContext
 
 from int total_facts, int total_size, int depth, float efficiency
 where 
-total_facts = strictcount(ControlFlowNode call, FunctionObject func |
+total_facts = strictcount(ControlFlowNode call, CallableValue func |
     exists(PointsToContext ctx |
-        call = PointsTo::get_a_call(func, ctx) and
+        call = func.getACall(ctx) and
         depth = ctx.getDepth()
     )
 )
 and
-total_size = strictcount(ControlFlowNode call, FunctionObject func, PointsToContext ctx |
-    call = PointsTo::get_a_call(func, ctx) and
+total_size = strictcount(ControlFlowNode call, CallableValue func, PointsToContext ctx |
+    call = func.getACall(ctx) and
     depth = ctx.getDepth()
 )
 and

python/ql/src/analysis/CallGraphMarginalEfficiency.ql

@@ -8,20 +8,20 @@ import semmle.python.pointsto.PointsToContext
 
 from int total_facts, int total_size, int depth, float efficiency
 where 
-total_facts = strictcount(ControlFlowNode call, FunctionObject func |
+total_facts = strictcount(ControlFlowNode call, CallableValue func |
     exists(PointsToContext ctx |
-        call = PointsTo::get_a_call(func, ctx) and
+        call = func.getACall(ctx) and
         depth = ctx.getDepth()
         and not
         exists(PointsToContext shallower |
-            call = PointsTo::get_a_call(func, shallower) and
+        call = func.getACall(shallower) and
             shallower.getDepth() < depth
         )
     )
 )
 and
-total_size = strictcount(ControlFlowNode call, FunctionObject func, PointsToContext ctx |
-    call = PointsTo::get_a_call(func, ctx) and
+total_size = strictcount(ControlFlowNode call, CallableValue func, PointsToContext ctx |
+    call = func.getACall(ctx) and
     depth = ctx.getDepth()
 )
 and

python/ql/src/analysis/ContextMarginalEfficiency.ql

@@ -29,4 +29,4 @@ total_size = strictcount(ControlFlowNode f, Object value, ClassObject cls, Point
 )
 and
 efficiency = 100.0 * total_facts / total_size
-select depth, total_facts, total_size, efficiency
+select depth, total_facts, total_size, efficiency

python/ql/src/analysis/FailedInference.ql

@@ -2,10 +2,10 @@
 import python
 import semmle.python.pointsto.PointsTo
 
-from ClassObject cls, string reason
+from ClassValue cls, string reason
 
 where
-PointsTo::Types::failed_inference(cls, reason)
+Types::failedInference(cls, reason)
 
 select cls, reason
 

python/ql/src/analysis/KeyPointsToFailure.ql

@@ -8,11 +8,12 @@
  */
 
 import python
+import semmle.python.pointsto.PointsTo
 
 predicate points_to_failure(Expr e) {
-    exists(ControlFlowNode f | 
+    exists(ControlFlowNode f |
         f = e.getAFlowNode() |
-        not f.refersTo(_)
+        not PointsTo::pointsTo(f, _, _, _)
     )
 }
 

python/ql/src/analysis/Pruned.ql

@@ -6,7 +6,7 @@ from int size
 
 where
 size = count(ControlFlowNode f |
-    not PointsTo::Test::reachableBlock(f.getBasicBlock(), _)
+    not PointsToInternal::reachableBlock(f.getBasicBlock(), _)
 )
 
 

python/ql/src/python.qll

@@ -36,5 +36,6 @@ import semmle.dataflow.SSA
 import semmle.python.pointsto.Base
 import semmle.python.pointsto.Context
 import semmle.python.pointsto.CallGraph
+import semmle.python.objects.ObjectAPI
 
 import site

python/ql/src/semmle/dataflow/SSA.qll

@@ -128,6 +128,14 @@ class EssaVariable extends TEssaDefinition {
         result = this.getDefinition().getScope()
     }
 
+    /** Holds if this the meta-variable for a scope.
+     * This is used to attach attributes for undeclared variables implicitly
+     * defined by `from ... import *` and the like.
+     */
+    predicate isMetaVariable() {
+        this.getName() = "$"
+    }
+
 }
 
 /* Helper for location_string 
@@ -332,8 +340,7 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
     }
 
     /** Gets the input variable for this phi node on the edge `pred` -> `this.getBasicBlock()`, if any. */
-    pragma [noinline]
-    EssaVariable getInput(BasicBlock pred) {
+    cached EssaVariable getInput(BasicBlock pred) {
         result.getDefinition() = this.reachingDefinition(pred)
         or
         result.getDefinition() = this.inputEdgeRefinement(pred)

python/ql/src/semmle/python/Exprs.qll

@@ -78,37 +78,28 @@ class Expr extends Expr_, AstNode {
      * NOTE: For complex dataflow, involving multiple stages of points-to analysis, it may be more precise to use 
      * `ControlFlowNode.refersTo(...)` instead.
      */
-    predicate refersTo(Object value, ClassObject cls, AstNode origin) {
-        not py_special_objects(cls, "_semmle_unknown_type")
-        and
-        not value = unknownValue()
-        and
-        PointsTo::points_to(this.getAFlowNode(), _, value, cls, origin.getAFlowNode())
+    predicate refersTo(Object obj, ClassObject cls, AstNode origin) {
+        this.refersTo(_, obj, cls, origin)
     }
 
     /** Gets what this expression might "refer-to" in the given `context`.
      */
-    predicate refersTo(Context context, Object value, ClassObject cls, AstNode origin) {
-        not py_special_objects(cls, "_semmle_unknown_type")
-        and
-        PointsTo::points_to(this.getAFlowNode(), context, value, cls, origin.getAFlowNode())
+    predicate refersTo(Context context, Object obj, ClassObject cls, AstNode origin) {
+        this.getAFlowNode().refersTo(context, obj, cls, origin.getAFlowNode())
     }
 
     /** Whether this expression might "refer-to" to `value` which is from `origin` 
      * Unlike `this.refersTo(value, _, origin)`, this predicate includes results 
      * where the class cannot be inferred.
      */
-    predicate refersTo(Object value, AstNode origin) {
-        PointsTo::points_to(this.getAFlowNode(), _, value, _, origin.getAFlowNode())
-        and
-        not value = unknownValue()
+    pragma[nomagic]
+    predicate refersTo(Object obj, AstNode origin) {
+        this.getAFlowNode().refersTo(obj, origin.getAFlowNode())
     }
 
     /** Equivalent to `this.refersTo(value, _)` */
-    predicate refersTo(Object value) {
-        PointsTo::points_to(this.getAFlowNode(), _, value, _, _)
-        and
-        not value = unknownValue()
+    predicate refersTo(Object obj) {
+        this.refersTo(obj, _)
     }
 
 }

python/ql/src/semmle/python/Flow.qll

@@ -212,47 +212,58 @@ class ControlFlowNode extends @py_flow_node {
         py_scope_flow(this, _, -1)
     }
 
-    /** Use ControlFlowNode.refersTo() instead. */
-    deprecated Object pointsTo() {
-        this.refersTo(result)
+    /** The value that this ControlFlowNode points-to. */
+    predicate pointsTo(Value value) {
+        this.pointsTo(_, value, _)
+    }
+
+    /** Gets a value that this ControlFlowNode may points-to. */
+    Value inferredValue() {
+        this.pointsTo(_, result, _)
+    }
+
+    /** The value and origin that this ControlFlowNode points-to. */
+    predicate pointsTo(Value value, ControlFlowNode origin) {
+        this.pointsTo(_, value, origin)
+    }
+
+    /** The value and origin that this ControlFlowNode points-to, given the context. */
+    predicate pointsTo(Context context, Value value, ControlFlowNode origin) {
+        PointsTo::pointsTo(this, context, value, origin)
     }
 
     /** Gets what this flow node might "refer-to". Performs a combination of localized (intra-procedural) points-to
      *  analysis and global module-level analysis. This points-to analysis favours precision over recall. It is highly
      *  precise, but may not provide information for a significant number of flow-nodes. 
      *  If the class is unimportant then use `refersTo(value)` or `refersTo(value, origin)` instead.
      */
-    predicate refersTo(Object value, ClassObject cls, ControlFlowNode origin) {
-        not py_special_objects(cls, "_semmle_unknown_type")
-        and
-        not value = unknownValue()
-        and
-        PointsTo::points_to(this, _, value, cls, origin)
+    pragma [nomagic]
+    predicate refersTo(Object obj, ClassObject cls, ControlFlowNode origin) {
+        this.refersTo(_, obj, cls, origin)
     }
 
     /** Gets what this expression might "refer-to" in the given `context`.
      */
-    predicate refersTo(Context context, Object value, ClassObject cls, ControlFlowNode origin) {
-        not py_special_objects(cls, "_semmle_unknown_type")
-        and
-        PointsTo::points_to(this, context, value, cls, origin)
+    pragma [nomagic]
+    predicate refersTo(Context context, Object obj, ClassObject cls, ControlFlowNode origin) {
+        not obj = unknownValue() and
+        not cls = theUnknownType() and
+        PointsTo::points_to(this, context, obj, cls, origin)
     }
 
     /** Whether this flow node might "refer-to" to `value` which is from `origin` 
      * Unlike `this.refersTo(value, _, origin)` this predicate includes results 
      * where the class cannot be inferred. 
      */
-    predicate refersTo(Object value, ControlFlowNode origin) {
-        PointsTo::points_to(this, _, value, _, origin)
-        and
-        not value = unknownValue()
+    pragma [nomagic]
+    predicate refersTo(Object obj, ControlFlowNode origin) {
+        not obj = unknownValue() and
+        PointsTo::points_to(this, _, obj, _, origin)
     }
 
     /** Equivalent to `this.refersTo(value, _)` */
-    predicate refersTo(Object value) {
-        PointsTo::points_to(this, _, value, _, _)
-        and
-        not value = unknownValue()
+    predicate refersTo(Object obj) {
+        this.refersTo(obj, _)
     }
 
     /** Gets the basic block containing this flow node */
@@ -460,6 +471,16 @@ class CallNode extends ControlFlowNode {
 
     override Call getNode() { result = super.getNode() }
 
+    predicate isDecoratorCall() {
+        exists(FunctionExpr func |
+            this.getNode() = func.getADecoratorCall()
+        )
+        or
+        exists(ClassExpr cls |
+            this.getNode() = cls.getADecoratorCall()
+        )
+    }
+
 }
 
 /** A control flow corresponding to an attribute expression, such as `value.attr` */
@@ -643,6 +664,16 @@ class BinaryExprNode extends ControlFlowNode {
         result = this.getNode().getOp()
     }
 
+    /** Whether left and right are a pair of operands for this binary expression */
+    predicate operands(ControlFlowNode left, Operator op, ControlFlowNode right) {
+        exists(BinaryExpr b, Expr eleft, Expr eright |
+            this.getNode() = b and left.getNode() = eleft and right.getNode() = eright  |
+            eleft = b.getLeft() and eright = b.getRight() and op = b.getOp()
+        ) and
+        left.getBasicBlock().dominates(this.getBasicBlock()) and
+        right.getBasicBlock().dominates(this.getBasicBlock())
+    }
+
 }
 
 /** A control flow node corresponding to a boolean shortcut (and/or) operation */
@@ -986,6 +1017,18 @@ class BasicBlock extends @py_flow_node {
         result = this.getLastNode().getAFalseSuccessor().getBasicBlock()
     }
 
+    /** Gets an unconditional successor to this basic block */
+    BasicBlock getAnUnconditionalSuccessor() {
+        result = this.getASuccessor() and
+        not result = this.getATrueSuccessor() and
+        not result = this.getAFalseSuccessor()
+    }
+
+    /** Gets an exceptional successor to this basic block */
+    BasicBlock getAnExceptionalSuccessor() {
+        result = this.getLastNode().getAnExceptionalSuccessor().getBasicBlock()
+    }
+
     /** Gets the scope of this block */
     pragma [nomagic] Scope getScope() {
         exists(ControlFlowNode n |
@@ -1037,6 +1080,22 @@ class BasicBlock extends @py_flow_node {
         not result.(ConditionBlock).controls(this, _)
     }
 
+    /** Holds if flow from this BasicBlock always reaches `succ`
+     */
+    predicate alwaysReaches(BasicBlock succ) {
+        succ = this
+        or
+        strictcount(this.getASuccessor()) = 1
+        and succ = this.getASuccessor()
+        or
+        forex(BasicBlock immsucc |
+            immsucc = this.getASuccessor()
+            |
+            immsucc.alwaysReaches(succ)
+        )
+
+    }
+
 }
 
 private predicate start_bb_likely_reachable(BasicBlock b) {

python/ql/src/semmle/python/Import.qll

@@ -14,7 +14,7 @@ class Alias extends Alias_ {
 private predicate valid_module_name(string name) {
     exists(Module m | m.getName() = name)
     or
-    exists(Builtin cmod | cmod.getClass() = theModuleType().asBuiltin() and cmod.getName() = name)
+    exists(Builtin cmod | cmod.getClass() = Builtin::special("ModuleType") and cmod.getName() = name)
 }
 
 /** An artificial expression representing an import */