From c48d63f2ec1a9b14e7693783cc71607b1094d61e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 19 Mar 2019 14:47:00 +0000 Subject: [PATCH 001/128] Python: First draft of ADT based objects and attendant points-to. --- python/ql/src/semmle/python/Module.qll | 2 +- .../ql/src/semmle/python/objects/TObject.qll | 981 ++++++++++++++++++ .../src/semmle/python/pointsto/PointsTo2.qll | 680 ++++++++++++ .../python/pointsto/PointsToContext2.qll | 276 +++++ .../PointsTo/local/LocalPointsTo.expected | 13 + .../PointsTo/local/LocalPointsTo.ql | 10 + .../test/library-tests/PointsTo/local/test.py | 44 + 7 files changed, 2005 insertions(+), 1 deletion(-) create mode 100644 python/ql/src/semmle/python/objects/TObject.qll create mode 100644 python/ql/src/semmle/python/pointsto/PointsTo2.qll create mode 100644 python/ql/src/semmle/python/pointsto/PointsToContext2.qll create mode 100644 python/ql/test/library-tests/PointsTo/local/LocalPointsTo.expected create mode 100644 python/ql/test/library-tests/PointsTo/local/LocalPointsTo.ql create mode 100644 python/ql/test/library-tests/PointsTo/local/test.py diff --git a/python/ql/src/semmle/python/Module.qll b/python/ql/src/semmle/python/Module.qll index 43000e356478..ec62088cc94e 100644 --- a/python/ql/src/semmle/python/Module.qll +++ b/python/ql/src/semmle/python/Module.qll @@ -211,7 +211,7 @@ private string moduleNameFromBase(Container file) { file instanceof File and result = file.getStem() } -private string moduleNameFromFile(Container file) { +string moduleNameFromFile(Container file) { exists(string basename | basename = moduleNameFromBase(file) and legalShortName(basename) diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll new file mode 100644 index 000000000000..812026ed8905 --- /dev/null +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -0,0 +1,981 @@ +import python +private import semmle.python.types.Builtins +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsToContext2 + +newtype TObject = + TBuiltinClassObject(Builtin bltn) { + bltn.isClass() and not bltn = Builtin::unknownType() + } + or + TBuiltinFunctionObject(Builtin bltn) { bltn.isFunction() } + or + TBuiltinMethodObject(Builtin bltn) { bltn.isMethod() } + or + TBuiltinModuleObject(Builtin bltn) { bltn.isModule() } + or + TBuiltinOpaqueObject(Builtin bltn) { + not bltn.isClass() and not bltn.isFunction() and + not bltn.isMethod() and not bltn.isModule() and + not py_special_objects(bltn, _) + } + or + TPythonFunctionObject(ControlFlowNode callable) { + callable.getNode() instanceof CallableExpr + } + or + TPythonClassObject(ControlFlowNode classdef) { + classdef.getNode() instanceof ClassDef + } + or + TPackageObject(Folder f) + or + TPythonModule(Module m) { not m.isPackage() } + or + TTrue() + or + TFalse() + or + TNone() + or + TUnknown() + or + TUnknownClass() + or + TUndefined() + + +library class ClassDecl extends @py_object { + + ClassDecl() { + this.(Builtin).isClass() and not this = Builtin::unknownType() + or + this.(ControlFlowNode).getNode() instanceof ClassDef + } + + string toString() { + result = "ClassDecl" + } + + predicate declaresAttribute(string name) { + exists(this.(Builtin).getMember(name)) + or + exists(Class cls | + cls = this.(ControlFlowNode).getNode().(ClassDef).getDefinedClass() and + exists(SsaVariable var | name = var.getId() and var.getAUse() = cls.getANormalExit()) + ) + } +} + +abstract class ObjectInternal extends TObject { + + abstract string toString(); + + /** The boolean value of this object, if it has one */ + abstract boolean booleanValue(); + + /** Holds if this object may be true or false when evaluated as a bool */ + abstract predicate maybe(); + + abstract predicate instantiated(ControlFlowNode node, PointsToContext2 context); + + /** Gets the class declaration for this object, if it is a declared class. */ + abstract ClassDecl getClassDeclaration(); + + abstract predicate isClass(); + + abstract predicate notClass(); + + abstract ObjectInternal getClass(); + + /** Holds if whatever this "object" represents can be meaningfully analysed for + * truth or false in comparisons. For example, `None` or `int` can be, but `int()` + * or an unknown string cannot. + */ + abstract predicate isComparable(); + + /** The negation of `isComparable()` */ + abstract predicate notComparable(); + + /** Gets the `Builtin` for this object, if any. + * All objects (except unknown and undefined values) should return + * exactly one result for either this method or `getOrigin()`. + */ + abstract Builtin getBuiltin(); + + /** Gets a control flow node that represents the source origin of this + * objects. + * All objects (except unknown and undefined values) should return + * exactly one result for either this method or `getBuiltin()`. + */ + abstract ControlFlowNode getOrigin(); + + /** Holds if `obj` is the result of calling `this` and `origin` is + * the origin of `obj`. + */ + abstract predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin); + + predicate hasLocationInfo(string fp, int bl, int bc, int el, int ec) { + this.getOrigin().getLocation().hasLocationInfo(fp, bl, bc, el, ec) + } + +} + +class PythonFunctionObjectInternal extends ObjectInternal, TPythonFunctionObject { + + Function getScope() { + exists(CallableExpr expr | + this = TPythonFunctionObject(expr.getAFlowNode()) and + result = expr.getInnerScope() + ) + } + + override string toString() { + result = this.getScope().toString() + } + + /** The boolean value of this object, if it has one */ + override boolean booleanValue() { + result = true + } + + /** Holds if this object may be true or false when evaluated as a bool */ + override predicate maybe() { none() } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + this = TPythonFunctionObject(node) and context.appliesTo(node) + } + + /** INTERNAL */ + override ClassDecl getClassDeclaration() { none() } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("FunctionType")) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Builtin getBuiltin() { + none() + } + + override ControlFlowNode getOrigin() { + this = TPythonFunctionObject(result) + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + exists(Function func, ControlFlowNode rval | + func = this.getScope() and + callee.appliesToScope(func) and + rval = func.getAReturnValueFlowNode() and + PointsTo2::points_to(rval, callee, obj, origin) + ) + } + +} + + +abstract class ClassObjectInternal extends ObjectInternal { + + override boolean booleanValue() { + result = true + } + + override predicate maybe() { none() } + + override predicate isClass() { any() } + + override predicate notClass() { none() } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // TO DO .. Result should (in most cases) be an instance + none() + } + +} + +class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject { + + Class getScope() { + exists(ClassDef def | + this = TPythonClassObject(def.getAFlowNode()) and + result = def.getDefinedClass() + ) + } + + override string toString() { + result = this.getScope().toString() + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + exists(DefinitionNode def | + this = TPythonClassObject(def) and + node = def.getValue() and + context.appliesTo(node) + ) + } + + override ClassDecl getClassDeclaration() { + this = TPythonClassObject(result) + } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("FunctionType")) + } + + override Builtin getBuiltin() { + none() + } + + override ControlFlowNode getOrigin() { + this = TPythonClassObject(result) + } + +} + +class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObject { + + override Builtin getBuiltin() { + this = TBuiltinClassObject(result) + } + + override string toString() { + result = "builtin class " + this.getBuiltin().getName() + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override ClassDecl getClassDeclaration() { + this = TBuiltinClassObject(result) + } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override ControlFlowNode getOrigin() { + none() + } + +} + +abstract class ModuleObjectInternal extends ObjectInternal { + + abstract string getName(); + + abstract Module getSourceModule(); + + abstract predicate isBuiltin(); + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // Modules aren't callable + none() + } + + override ControlFlowNode getOrigin() { + result = this.getSourceModule().getEntryNode() + } + +} + +class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleObject { + + override Builtin getBuiltin() { + this = TBuiltinModuleObject(result) + } + + override string toString() { + result = "builtin module " + this.getBuiltin().getName() + } + + override string getName() { + result = this.getBuiltin().getName() + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override boolean booleanValue() { + result = true + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Module getSourceModule() { + none() + } + + override predicate isBuiltin() { + any() + } + +} + +class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { + + override Builtin getBuiltin() { + none() + } + + Folder getFolder() { + this = TPackageObject(result) + } + + override string toString() { + result = "package " + this.getName() + } + + override string getName() { + result = moduleNameFromFile(this.getFolder()) + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override boolean booleanValue() { + result = true + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Module getSourceModule() { + result.getFile() = this.getFolder().getFile("__init__.py") + } + + PythonModuleObjectInternal getInitModule() { + result = TPythonModule(this.getSourceModule()) + } + + predicate hasNoInitModule() { + exists(Folder f | + f = this.getFolder() and + not exists(f.getFile("__init__.py")) + ) + } + + override predicate isBuiltin() { + none() + } + + ModuleObjectInternal submodule(string name) { + result.getName() = this.getName() + "." + name + } + + override predicate hasLocationInfo(string fp, int bl, int bc, int el, int ec) { + this.getOrigin().getLocation().hasLocationInfo(fp, bl, bc, el, ec) + or + this.hasNoInitModule() and fp = this.getFolder().getAbsolutePath() and + bl = 0 and bc = 0 and el = 0 and ec = 0 + } + +} + +class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { + + override Builtin getBuiltin() { + none() + } + + override string toString() { + result = "package " + this.getName() + } + + override string getName() { + result = this.getSourceModule().getName() + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override boolean booleanValue() { + result = true + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Module getSourceModule() { + this = TPythonModule(result) + } + + PythonModuleObjectInternal getInitModule() { + result = TPythonModule(this.getSourceModule()) + } + + override predicate isBuiltin() { + none() + } + +} + + +class BuiltinFunctionObjectInternal extends ObjectInternal, TBuiltinFunctionObject { + + override Builtin getBuiltin() { + this = TBuiltinFunctionObject(result) + } + + override string toString() { + result = "builtin function " + this.getBuiltin().getName() + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override boolean booleanValue() { + result = true + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. + none() + } + + override ControlFlowNode getOrigin() { + none() + } + +} + + +class BuiltinMethodObjectInternal extends ObjectInternal, TBuiltinMethodObject { + + override Builtin getBuiltin() { + this = TBuiltinMethodObject(result) + } + + override string toString() { + result = "builtin method " + this.getBuiltin().getName() + } + + override boolean booleanValue() { + result = true + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. + none() + } + + override ControlFlowNode getOrigin() { + none() + } + +} + + +class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { + + override Builtin getBuiltin() { + this = TBuiltinOpaqueObject(result) + } + + override string toString() { + none() + } + + override boolean booleanValue() { + // TO DO ... Depends on class. `this.getClass().instancesAlways(result)` + none() + } + + override predicate maybe() { + // TO DO ... Depends on class. `this.getClass().instancesMaybe()` + any() + } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate isComparable() { + none() + } + + override predicate notComparable() { + any() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and + callee_for_object(callee, this) + } + + override ControlFlowNode getOrigin() { + none() + } + +} + +private predicate callee_for_object(PointsToContext2 callee, ObjectInternal obj) { + exists(CallNode call, PointsToContext2 caller | + callee.fromCall(call, caller) and + PointsTo2::points_to(call.getFunction(), caller, obj, _) + ) +} + + +abstract class BooleanObjectInternal extends ObjectInternal { + + BooleanObjectInternal() { + this = TTrue() or this = TFalse() + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("bool")) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Builtin getBuiltin() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // Booleans aren't callable + none() + } + + override ControlFlowNode getOrigin() { + none() + } + +} + +class TrueObjectInternal extends BooleanObjectInternal, TTrue { + + override string toString() { + result = "True" + } + + override boolean booleanValue() { + result = true + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + node.(NameNode).getId() = "True" and context.appliesTo(node) + } + +} + +class FalseObjectInternal extends BooleanObjectInternal, TFalse { + + override string toString() { + result = "False" + } + + override boolean booleanValue() { + result = false + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + node.(NameNode).getId() = "False" and context.appliesTo(node) + } + +} + +class NoneObjectInternal extends ObjectInternal, TNone { + + override string toString() { + result = "None" + } + + override boolean booleanValue() { + result = false + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("NoneType")) + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + node.(NameNode).getId() = "None" and context.appliesTo(node) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Builtin getBuiltin() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // None isn't callable + none() + } + + override ControlFlowNode getOrigin() { + none() + } + +} + + +class UnknownInternal extends ObjectInternal, TUnknown { + + override string toString() { + none() + } + + override boolean booleanValue() { + none() + } + + override predicate maybe() { any() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TUnknownClass() + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate isComparable() { + none() + } + + override predicate notComparable() { + any() + } + + override Builtin getBuiltin() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and + callee_for_object(callee, this) + } + + override ControlFlowNode getOrigin() { + none() + } + +} + +class UnknownClassInternal extends ObjectInternal, TUnknownClass { + + override string toString() { + none() + } + + override boolean booleanValue() { + none() + } + + override predicate maybe() { any() } + + override ClassDecl getClassDeclaration() { + result = Builtin::unknownType() + } + + override predicate isClass() { any() } + + override predicate notClass() { none() } + + override ObjectInternal getClass() { + result = TUnknownClass() + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate isComparable() { + none() + } + + override predicate notComparable() { + any() + } + + override Builtin getBuiltin() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and + callee_for_object(callee, this) + } + + override ControlFlowNode getOrigin() { + none() + } + +} + +class UndefinedInternal extends ObjectInternal, TUndefined { + + override string toString() { + none() + } + + override boolean booleanValue() { + none() + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + none() + } + + override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate isComparable() { + none() + } + + override predicate notComparable() { + any() + } + + override Builtin getBuiltin() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // Accessing an undefined value raises a NameError, but if during import it probably + // means that we missed an import. + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and + callee.getOuter().isImport() + } + + override ControlFlowNode getOrigin() { + none() + } + +} + +module ObjectInternal { + + ObjectInternal bool(boolean b) { + b = true and result = TTrue() + or + b = false and result = TFalse() + } + + ObjectInternal none_() { + result = TNone() + } + + + ObjectInternal unknown() { + result = TUnknown() + } + + ObjectInternal unknownClass() { + result = TUnknownClass() + } + + ObjectInternal undefined() { + result = TUndefined() + } + + ObjectInternal builtin(string name) { + result = TBuiltinClassObject(Builtin::builtin(name)) + or + result = TBuiltinFunctionObject(Builtin::builtin(name)) + or + result = TBuiltinOpaqueObject(Builtin::builtin(name)) + } + + ObjectInternal sysModules() { + result = TBuiltinOpaqueObject(Builtin::special("sys").getMember("modules")) + } + +} + diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll new file mode 100644 index 000000000000..d2e55dcc3b57 --- /dev/null +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -0,0 +1,680 @@ +import python + +private import semmle.python.objects.TObject +private import semmle.python.pointsto.Filters +private import semmle.python.pointsto.PointsToContext2 + +/* Use this version for speed */ +library class CfgOrigin extends @py_object { + + string toString() { + /* Not to be displayed */ + none() + } + + /** Get a `ControlFlowNode` from `this` or `here`. + * If `this` is a ControlFlowNode then use that, otherwise fall back on `here` + */ + pragma[inline] + ControlFlowNode asCfgNodeOrHere(ControlFlowNode here) { + result = this + or + not this instanceof ControlFlowNode and result = here + } + + ControlFlowNode toCfgNode() { + result = this + } + + pragma[inline] + CfgOrigin fix(ControlFlowNode here) { + if this = unknownValue() then + result = here + else + result = this + } + +} + +/* Use this version for stronger type-checking */ +//private newtype TCfgOrigin = +// TUnknownOrigin() +// or +// TCfgOrigin(ControlFlowNode f) +// +//library class CfgOrigin extends TCfgOrigin { +// +// string toString() { +// /* Not to be displayed */ +// none() +// } +// +// /** Get a `ControlFlowNode` from `this` or `here`. +// * If `this` is a ControlFlowNode then use that, otherwise fall back on `here` +// */ +// pragma[inline] +// ControlFlowNode asCfgNodeOrHere(ControlFlowNode here) { +// this = TUnknownOrigin() and result = here +// or +// this = TCfgOrigin(result) +// } +// +// ControlFlowNode toCfgNode() { +// this = TCfgOrigin(result) +// } +// +// CfgOrigin fix(ControlFlowNode here) { +// this = TUnknownOrigin() and result = TCfgOrigin(here) +// or +// not this = TUnknownOrigin() and result = this +// } +//} +// + + +module CfgOrigin { + + CfgOrigin fromCfgNode(ControlFlowNode f) { + result = f + } + + CfgOrigin unknown() { + result = unknownValue() + } + +} + +module PointsTo2 { + + /** INTERNAL -- Use `f.refersTo(value, origin)` instead. */ + predicate points_to(ControlFlowNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + points_to_candidate(f, context, value, origin) and + reachableBlock(f.getBasicBlock(), context) + } + + predicate points_to_candidate(ControlFlowNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + use_points_to(f, context, value, origin) + or + subscript_points_to(f, context, value, origin) + or + binary_expr_points_to(f, context, value, origin) + or + origin = f and compare_expr_points_to(f, context, value) + or + origin = f and not_points_to(f, context, value) + or + origin = f and value.instantiated(f, context) + or + InterModulePointsTo::import_points_to(f, context, value, origin) + or + InterModulePointsTo::from_import_points_to(f, context, value, origin) + or + InterProceduralPointsTo::call_points_to(f, context, value, origin) + // To do... More stuff here :) + // or + // f.(CustomPointsToFact).pointsTo(context, value, origin) + } + + cached CallNode get_a_call(ObjectInternal func, PointsToContext2 context) { + points_to(result.getFunction(), context, func, _) + } + + /* Holds if BasicBlock `b` is reachable, given the context `context`. */ + predicate reachableBlock(BasicBlock b, PointsToContext2 context) { + context.appliesToScope(b.getScope()) and not exists(ConditionBlock guard | guard.controls(b, _)) + or + exists(ConditionBlock guard | + guard = b.getImmediatelyControllingBlock() and + reachableBlock(guard, context) + | + exists(ObjectInternal value | + points_to(guard.getLastNode(), context, value, _) + | + guard.controls(b, _) and value.maybe() + or + guard.controls(b, true) and value.booleanValue() = true + or + guard.controls(b, false) and value.booleanValue() = false + ) + or + /* Assume the true edge of an assert is reachable (except for assert 0/False) */ + guard.controls(b, true) and + exists(Assert a, Expr test | + a.getTest() = test and + guard.getLastNode().getNode() = test and + not test instanceof ImmutableLiteral + ) + ) + } + + /* Holds if the edge `pred` -> `succ` is reachable, given the context `context`. + */ + predicate controlledReachableEdge(BasicBlock pred, BasicBlock succ, PointsToContext2 context) { + exists(ConditionBlock guard, ObjectInternal value | + points_to(guard.getLastNode(), context, value, _) + | + guard.controlsEdge(pred, succ, _) and value.maybe() + or + guard.controlsEdge(pred, succ, true) and value.booleanValue() = true + or + guard.controlsEdge(pred, succ, false) and value.booleanValue() = false + ) + } + + /** Gets an object pointed to by a use (of a variable). */ + private predicate use_points_to(NameNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + exists(CfgOrigin origin_or_obj | + value != ObjectInternal::undefined() and + use_points_to_maybe_origin(f, context, value, origin_or_obj) | + origin = origin_or_obj.asCfgNodeOrHere(f) + ) + } + + private predicate use_points_to_maybe_origin(NameNode f, PointsToContext2 context, ObjectInternal value, CfgOrigin origin_or_obj) { + ssa_variable_points_to(fast_local_variable(f), context, value, origin_or_obj) + or + name_lookup_points_to_maybe_origin(f, context, value, origin_or_obj) + or + not exists(fast_local_variable(f)) and not exists(name_local_variable(f)) and + global_lookup_points_to_maybe_origin(f, context, value, origin_or_obj) + } + + /** Holds if `var` refers to `(value, origin)` given the context `context`. */ + predicate ssa_variable_points_to(EssaVariable var, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + ssa_definition_points_to(var.getDefinition(), context, value, origin) + } + + private predicate name_lookup_points_to_maybe_origin(NameNode f, PointsToContext2 context, ObjectInternal value, CfgOrigin origin_or_obj) { + exists(EssaVariable var | var = name_local_variable(f) | + ssa_variable_points_to(var, context, value, origin_or_obj) + ) + or + local_variable_undefined(f, context) and + global_lookup_points_to_maybe_origin(f, context, value, origin_or_obj) + } + + pragma [noinline] + private predicate local_variable_undefined(NameNode f, PointsToContext2 context) { + ssa_variable_points_to(name_local_variable(f), context, ObjectInternal::undefined(), _) + } + + private predicate global_lookup_points_to_maybe_origin(NameNode f, PointsToContext2 context, ObjectInternal value, CfgOrigin origin_or_obj) { + ssa_variable_points_to(global_variable(f), context, value, origin_or_obj) + or + exists(ControlFlowNode origin | + origin_or_obj = CfgOrigin::fromCfgNode(origin) + | + ssa_variable_points_to(global_variable(f), context, ObjectInternal::undefined(), _) and + potential_builtin_points_to(f, value, origin) + or + not exists(global_variable(f)) and context.appliesToScope(f.getScope()) and + potential_builtin_points_to(f, value, origin) + ) + } + + /** The ESSA variable with fast-local lookup (LOAD_FAST bytecode). */ + private EssaVariable fast_local_variable(NameNode n) { + n.isLoad() and + result.getASourceUse() = n and + result.getSourceVariable() instanceof FastLocalVariable + } + + /** The ESSA variable with name-local lookup (LOAD_NAME bytecode). */ + private EssaVariable name_local_variable(NameNode n) { + n.isLoad() and + result.getASourceUse() = n and + result.getSourceVariable() instanceof NameLocalVariable + } + + /** The ESSA variable for the global variable lookup. */ + private EssaVariable global_variable(NameNode n) { + n.isLoad() and + result.getASourceUse() = n and + result.getSourceVariable() instanceof GlobalVariable + } + + /** Holds if the ESSA definition `def` refers to `(value, origin)` given the context `context`. */ + predicate ssa_definition_points_to(EssaDefinition def, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + ssa_phi_points_to(def, context, value, origin) + or + exists(ControlFlowNode orig | + ssa_node_definition_points_to(def, context, value, orig) and + origin = CfgOrigin::fromCfgNode(orig) + ) + or + ssa_filter_definition_points_to(def, context, value, origin) + or + ssa_node_refinement_points_to(def, context, value, origin) + } + + pragma [noinline] + private predicate ssa_node_definition_points_to(EssaNodeDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + reachableBlock(def.getDefiningNode().getBasicBlock(), _) and + ssa_node_definition_points_to_unpruned(def, context, value, origin) + } + + pragma [nomagic] + private predicate ssa_node_definition_points_to_unpruned(EssaNodeDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + parameter_points_to(def, context, value, origin) + or + assignment_points_to(def, context, value, origin) + //// TO DO... + // or + // self_parameter_points_to(def, context, value, origin) + // or + // delete_points_to(def, context, value, origin) + // or + // module_name_points_to(def, context, value, origin) + or + scope_entry_points_to(def, context, value, origin) + // or + // implicit_submodule_points_to(def, context, value, origin) + // or + // iteration_definition_points_to(def, context, value, origin) + /* + * No points-to for non-local function entry definitions yet. + */ + } + + pragma [noinline] + private predicate ssa_node_refinement_points_to(EssaNodeRefinement def, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + //method_callsite_points_to(def, context, value, origin) + //or + //import_star_points_to(def, context, value, origin) + //or + //attribute_assignment_points_to(def, context, value, origin) + //or + //callsite_points_to(def, context, value, origin) + //or + //argument_points_to(def, context, value, origin) + //or + //attribute_delete_points_to(def, context, value, origin) + //or + uni_edged_phi_points_to(def, context, value, origin) + } + + /** Holds if ESSA edge refinement, `def`, refers to `(value, cls, origin)`. */ + predicate ssa_filter_definition_points_to(PyEdgeRefinement def, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + exists(ControlFlowNode test, ControlFlowNode use | + refinement_test(test, use, Conditionals::evaluatesTo(test, use, context, value, origin.toCfgNode()), def) + ) + } + + /** Holds if ESSA definition, `uniphi`, refers to `(value, origin)`. */ + pragma [noinline] + predicate uni_edged_phi_points_to(SingleSuccessorGuard uniphi, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + exists(ControlFlowNode test, ControlFlowNode use | + /* Because calls such as `len` may create a new variable, we need to go via the source variable + * That is perfectly safe as we are only dealing with calls that do not mutate their arguments. + */ + use = uniphi.getInput().getSourceVariable().(Variable).getAUse() and + test = uniphi.getDefiningNode() and + uniphi.getSense() = Conditionals::evaluatesTo(test, use, context, value, origin.toCfgNode()) + ) + } + + /** Points-to for normal assignments `def = ...`. */ + pragma [noinline] + private predicate assignment_points_to(AssignmentDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + points_to(def.getValue(), context, value, origin) + } + + + /** Points-to for parameter. `def foo(param): ...`. */ + pragma [noinline] + private predicate parameter_points_to(ParameterDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + positional_parameter_points_to(def, context, value, origin) + //// TO DO... + // or + // named_parameter_points_to(def, context, value, origin) + // or + // default_parameter_points_to(def, context, value, origin) + // or + // special_parameter_points_to(def, context, value, origin) + } + + /** Helper for `parameter_points_to` */ + pragma [noinline] + private predicate positional_parameter_points_to(ParameterDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + /// To do... + //exists(PointsToContext2 caller, ControlFlowNode arg | + // points_to(arg, caller, value, origin) and + // Flow::callsite_argument_transfer(arg, caller, def, context) + //) + //or + not def.isSelf() and not def.getParameter().isVarargs() and not def.getParameter().isKwargs() and + context.isRuntime() and value = ObjectInternal::unknown() and origin = def.getDefiningNode() + } + + /** Holds if the phi-function `phi` refers to `(value, origin)` given the context `context`. */ + pragma [nomagic] + private predicate ssa_phi_points_to(PhiFunction phi, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + exists(EssaVariable input, BasicBlock pred | + input = phi.getInput(pred) and + ssa_variable_points_to(input, context, value, origin) + | + controlledReachableEdge(pred, phi.getBasicBlock(), context) + or + not exists(ConditionBlock guard | guard.controlsEdge(pred, phi.getBasicBlock(), _)) + ) + or + ssa_variable_points_to(phi.getShortCircuitInput(), context, value, origin) + } + + /** Points-to for implicit variable declarations at scope-entry. */ + pragma [noinline] + private predicate scope_entry_points_to(ScopeEntryDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + /* Transfer from another scope */ + /// To do... + //exists(EssaVariable var, PointsToContext2 outer, CfgOrigin orig | + // Flow::scope_entry_value_transfer(var, outer, def, context) and + // ssa_variable_points_to(var, outer, value, orig) and + // origin = orig.asCfgNodeOrHere(def.getDefiningNode()) + //) + //or + /* Undefined variable */ + exists(Scope scope | + not def.getVariable().getName() = "__name__" and + not def.getVariable().getName() = "$" and + def.getScope() = scope and context.appliesToScope(scope) | + def.getSourceVariable() instanceof GlobalVariable and scope instanceof Module + or + def.getSourceVariable() instanceof LocalVariable and (context.isImport() or context.isRuntime() or context.isMain()) + ) and + value = ObjectInternal::undefined() and origin = def.getDefiningNode() + or + /* Builtin not defined in outer scope */ + exists(Module mod, GlobalVariable var | + var = def.getSourceVariable() and + mod = def.getScope().getEnclosingModule() and + context.appliesToScope(def.getScope()) and + not exists(EssaVariable v | v.getSourceVariable() = var and v.getScope() = mod) and + value = ObjectInternal::builtin(var.getId()) and origin = def.getDefiningNode() + ) + } + + private predicate subscript_points_to(SubscriptNode sub, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + points_to(sub.getValue(), context, ObjectInternal::unknown(), _) and + value = ObjectInternal::unknown() and origin = sub + } + + /** Track bitwise expressions so we can handle integer flags and enums. + * Tracking too many binary expressions is likely to kill performance. + */ + private predicate binary_expr_points_to(BinaryExprNode b, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + // TO DO... + // Track some integer values through `|` and the types of some objects + none() + } + + + private predicate compare_expr_points_to(CompareNode cmp, PointsToContext2 context, ObjectInternal value) { + exists(ControlFlowNode a, ControlFlowNode b, ObjectInternal o1, ObjectInternal o2 | + exists(boolean is | + equality_test(cmp, a, is, b) and + points_to(a, context, o1, _) and + points_to(b, context, o2, _) | + (o1.isComparable() and o2.isComparable()) and + ( + o1 = o2 and value = ObjectInternal::bool(is) + or + o1 != o2 and value = ObjectInternal::bool(is.booleanNot()) + ) + or + (o1.notComparable() or o2.notComparable()) and + value = ObjectInternal::bool(_) + ) + // TO DO -- Comparison of int and string constants. Version tests and the like. + //or + //const_compare(cmp, context, comp, strict) + // exists(int comp, boolean strict | + // const_compare(cmp, context, comp, strict) + // | + // comp = -1 and value = theTrueObject() + // or + // comp = 0 and strict = false and value = ObjectInternal::bool(true) + // or + // comp = 0 and strict = true and value = ObjectInternal::bool(false) + // or + // comp = 1 and value = ObjectInternal::bool(false) + // ) + // or + // value = version_tuple_compare(cmp, context) + ) + } + + private predicate not_points_to(UnaryExprNode f, PointsToContext2 context, ObjectInternal value) { + f.getNode().getOp() instanceof Not and + exists(ObjectInternal operand | + points_to(f.getOperand(), context, operand, _) + | + operand.maybe() and value = ObjectInternal::bool(_) + or + operand.booleanValue() = false and value = ObjectInternal::bool(true) + or + operand.booleanValue() = true and value = ObjectInternal::bool(false) + ) + } + +} + +module InterModulePointsTo { + + predicate import_points_to(ControlFlowNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + exists(string name, ImportExpr i | + i.getAFlowNode() = f and i.getImportedModuleName() = name and + module_imported_as(value, name) and + origin = f and + context.appliesTo(f) + ) + } + + predicate from_import_points_to(ImportMemberNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + exists(string name, ModuleObjectInternal mod, CfgOrigin orig | + PointsTo2::points_to(f.getModule(name), context, mod, _) and + origin = orig.asCfgNodeOrHere(f) + | + // TO DO... $ variables. + //mod.getSourceModule() = f.getEnclosingModule() and + //not exists(EssaVariable var | var.getSourceVariable().getName() = name and var.getAUse() = f) and + //exists(EssaVariable dollar | + // isModuleStateVariable(dollar) and dollar.getAUse() = f and + // SSA::ssa_variable_named_attribute_points_to(dollar, context, name, value, orig) + //) + //or + (mod.getSourceModule() != f.getEnclosingModule() or mod.isBuiltin()) and + module_attribute_points_to(mod, name, value, orig) + ) + or + exists(EssaVariable var, CfgOrigin orig | + var = ssa_variable_for_module_attribute(f, context) and + PointsTo2::ssa_variable_points_to(var, context, value, orig) and + origin = orig.asCfgNodeOrHere(f) + ) + } + + private EssaVariable ssa_variable_for_module_attribute(ImportMemberNode f, PointsToContext2 context) { + exists(string name, ModuleObjectInternal mod, Module m | + mod.getSourceModule() = m and m = f.getEnclosingModule() and m = result.getScope() and + PointsTo2::points_to(f.getModule(name), context, mod, _) and + result.getSourceVariable().getName() = name and result.getAUse() = f + ) + } + + /* Holds if `import name` will import the module `m`. */ + private predicate module_imported_as(ModuleObjectInternal m, string name) { + /* Normal imports */ + m.getName() = name + or + /* sys.modules['name'] = m */ + exists(ControlFlowNode sys_modules_flow, ControlFlowNode n, ControlFlowNode mod | + /* Use previous points-to here to avoid slowing down the recursion too much */ + exists(SubscriptNode sub | + sub.getValue() = sys_modules_flow and + PointsTo2::points_to(sys_modules_flow, _, ObjectInternal::sysModules(), _) and + sub.getIndex() = n and + n.getNode().(StrConst).getText() = name and + sub.(DefinitionNode).getValue() = mod and + PointsTo2::points_to(mod, _, m, _) + ) + ) + } + + /** Holds if `mod.name` points to `(value, origin)`, where `mod` is a module object. */ + predicate module_attribute_points_to(ModuleObjectInternal mod, string name, ObjectInternal value, CfgOrigin origin) { + py_module_attributes(mod.getSourceModule(), name, value, origin) + or + package_attribute_points_to(mod, name, value, origin) + or + value.getBuiltin() = mod.getBuiltin().getMember(name) and + origin = CfgOrigin::unknown() + } + + /** Holds if `m.name` points to `(value, origin)`, where `m` is a (source) module. */ + cached predicate py_module_attributes(Module m, string name, ObjectInternal obj, CfgOrigin origin) { + exists(EssaVariable var, ControlFlowNode exit, PointsToContext2 imp | + exit = m.getANormalExit() and var.getAUse() = exit and + var.getSourceVariable().getName() = name and + PointsTo2::ssa_variable_points_to(var, imp, obj, origin) and + imp.isImport() and + obj != ObjectInternal::undefined() + ) + // TO DO, dollar variable... + //or + //not exists(EssaVariable var | var.getAUse() = m.getANormalExit() and var.getSourceVariable().getName() = name) and + //exists(EssaVariable var, PointsToContext2 imp | + // var.getAUse() = m.getANormalExit() and isModuleStateVariable(var) | + // PointsTo2::ssa_variable_named_attribute_points_to(var, imp, name, obj, origin) and + // imp.isImport() and obj != ObjectInternal::undefined() + //) + } + + /** Holds if `package.name` points to `(value, origin)`, where `package` is a package object. */ + cached predicate package_attribute_points_to(PackageObjectInternal package, string name, ObjectInternal value, CfgOrigin origin) { + py_module_attributes(package.getInitModule().getSourceModule(), name, value, origin) + or + // TO DO + //exists(Module init | + // init = package.getInitModule().getModule() and + // not exists(EssaVariable var | var.getAUse() = init.getANormalExit() and var.getSourceVariable().getName() = name) and + // exists(EssaVariable var, Context context | + // isModuleStateVariable(var) and var.getAUse() = init.getANormalExit() and + // context.isImport() and + // SSA::ssa_variable_named_attribute_points_to(var, context, name, undefinedVariable(), _, _) and + // origin = value and + // value = package.submodule(name) + // ) + //) + //or + package.hasNoInitModule() and + exists(ModuleObjectInternal mod | + mod = package.submodule(name) and + value = mod | + origin = CfgOrigin::fromCfgNode(mod.getSourceModule().getEntryNode()) + or + mod.isBuiltin() and origin = CfgOrigin::unknown() + ) + } + +} + +module InterProceduralPointsTo { + + predicate call_points_to(CallNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + exists(ObjectInternal func, PointsToContext2 callee, CfgOrigin resultOrigin | + callee.fromCall(f, context) and + PointsTo2::points_to(f.getFunction(), context, func, _) and + func.callResult(callee, value, resultOrigin) and + origin = resultOrigin.fix(f) + ) + } + +} + +/** Gets the `value, origin` that `f` would refer to if it has not been assigned some other value */ +pragma [noinline] +private predicate potential_builtin_points_to(NameNode f, ObjectInternal value, ControlFlowNode origin) { + f.isGlobal() and f.isLoad() and origin = f and + ( + value = ObjectInternal::builtin(f.getId()) + or + not exists(Object::builtin(f.getId())) and value = ObjectInternal::unknown() + ) +} + +/** Get the ESSA pseudo-variable used to retain module state + * during module initialization. Module attributes are handled + * as attributes of this variable, allowing the SSA form to track + * mutations of the module during its creation. + */ +private predicate isModuleStateVariable(EssaVariable var) { + var.getName() = "$" and var.getScope() instanceof Module +} + +module Conditionals { + + /** Holds if `expr` is the operand of a unary `not` expression. */ + private ControlFlowNode not_operand(ControlFlowNode expr) { + expr.(UnaryExprNode).getNode().getOp() instanceof Not and + result = expr.(UnaryExprNode).getOperand() + } + + private boolean maybe() { + result = true or result = false + } + + boolean evaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { + //result = isinstance_test_evaluates_boolean(expr, use, context, val, origin) + //or + //result = issubclass_test_evaluates_boolean(expr, use, context, val, origin) + //or + result = equalityEvaluatesTo(expr, use, context, val, origin) + or + //result = callable_test_evaluates_boolean(expr, use, context, val, origin) + //or + //result = hasattr_test_evaluates_boolean(expr, use, context, val, origin) + //or + result = evaluatesTo(not_operand(expr), use, context, val, origin).booleanNot() + or + //result = true and evaluates_int(expr, use, context, val, origin) != 0 + //or + //result = false and evaluates_int(expr, use, context, val, origin) = 0 + //or + result = boolEvaluatesTo(expr, use, context, val, origin) + } + + pragma [noinline] + private boolean boolEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { + contains_interesting_expression_within_test(expr, use) and + PointsTo2::points_to(use, context, val, origin) and + expr = use and + ( + val.booleanValue() = result + or + val.maybe() and result = maybe() + ) + } + + pragma [noinline] + private boolean equalityEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { + exists(ControlFlowNode r, boolean sense | + contains_interesting_expression_within_test(expr, use) | + equality_test(expr, use, sense, r) and + exists(ObjectInternal other | + PointsTo2::points_to(use, context, val, origin) and + PointsTo2::points_to(r, context, other, _) | + val.isComparable() and other.isComparable() and + ( + other = val and result = sense + or + other != val and result = sense.booleanNot() + ) + or + val.notComparable() and result = maybe() + or + other.notComparable() and result = maybe() + ) + ) + } + +} diff --git a/python/ql/src/semmle/python/pointsto/PointsToContext2.qll b/python/ql/src/semmle/python/pointsto/PointsToContext2.qll new file mode 100644 index 000000000000..622213c40ecf --- /dev/null +++ b/python/ql/src/semmle/python/pointsto/PointsToContext2.qll @@ -0,0 +1,276 @@ +import python +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.objects.TObject +/* + * A note on 'cost'. Cost doesn't represent the cost to compute, + * but (a vague estimate of) the cost to compute per value gained. + * This is constantly evolving, so see the various cost functions below for more details. + */ + +private int given_cost() { + exists(string depth | + py_flags_versioned("context.cost", depth, _) and + result = depth.toInt() + ) +} + +private int max_context_cost() { + not py_flags_versioned("context.cost", _, _) and result = 7 + or + result = max(int cost | cost = given_cost() | cost) +} + +private int syntactic_call_count(Scope s) { + exists(Function f | + f = s and f.getName() != "__init__" | + result = count(CallNode call | + call.getFunction().(NameNode).getId() = f.getName() + or + call.getFunction().(AttrNode).getName() = f.getName() + ) + ) + or + s.getName() = "__init__" and result = 1 + or + not s instanceof Function and result = 0 +} + +private int incoming_call_cost(Scope s) { + /* Syntactic call count will often be a considerable overestimate + * of the actual number of calls, so we use the square root. + * Cost = log(sqrt(call-count)) + */ + result = ((syntactic_call_count(s)+1).log(2)*0.5).floor() +} + +private int context_cost(TPointsToContext ctx) { + ctx = TMainContext() and result = 0 + or + ctx = TRuntimeContext() and result = 0 + or + ctx = TImportContext() and result = 0 + or + ctx = TCallContext(_, _, result) +} + +private int call_cost(CallNode call) { + if call.getScope().inSource() then + result = 2 + else + result = 3 +} + +private int outgoing_calls(Scope s) { + result = strictcount(CallNode call | call.getScope() = s) +} + +predicate super_method_call(CallNode call) { + call.getFunction().(AttrNode).getObject().(CallNode).getFunction().(NameNode).getId() = "super" +} + +private int outgoing_call_cost(CallNode c) { + /* Cost = log(outgoing-call-count) */ + result = outgoing_calls(c.getScope()).log(2).floor() +} + +/** Cost of contexts for a call, the more callers the + * callee of call has the more expensive it is to add contexts for it. + * This seems to be an effective heuristics for preventing an explosion + * in the number of contexts while retaining good results. + */ +private int splay_cost(CallNode c) { + if super_method_call(c) then + result = 0 + else + result = outgoing_call_cost(c) + incoming_call_cost(c.getScope()) +} + +private predicate call_to_init_or_del(CallNode call) { + exists(string mname | + mname = "__init__" or mname = "__del__" | + mname = call.getFunction().(AttrNode).getName() + ) +} + +/** Total cost estimate */ +private int total_call_cost(CallNode call) { + /* We want to always follow __init__ and __del__ calls as they tell us about object construction, + * but we need to be aware of cycles, so they must have a non-zero cost. + */ + if call_to_init_or_del(call) then + result = 1 + else + result = call_cost(call) + splay_cost(call) +} + +private int total_cost(CallNode call, PointsToContext2 ctx) { + ctx.appliesTo(call) and + result = total_call_cost(call) + context_cost(ctx) +} + +private cached newtype TPointsToContext = + TMainContext() + or + TRuntimeContext() + or + TImportContext() + or + TCallContext(ControlFlowNode call, PointsToContext2 outerContext, int cost) { + total_cost(call, outerContext) = cost and + cost <= max_context_cost() + } + +/** Points-to context. Context can be one of: + * * "main": Used for scripts. + * * "import": Use for non-script modules. + * * "default": Use for functions and methods without caller context. + * * All other contexts are call contexts and consist of a pair of call-site and caller context. + */ +class PointsToContext2 extends TPointsToContext { + + cached string toString() { + this = TMainContext() and result = "main" + or + this = TRuntimeContext() and result = "runtime" + or + this = TImportContext() and result = "import" + or + exists(CallNode callsite, PointsToContext2 outerContext | + this = TCallContext(callsite, outerContext, _) and + result = callsite.getLocation() + " from " + outerContext.toString() + ) + } + + /** Holds if `call` is the call-site from which this context was entered and `outer` is the caller's context. */ + predicate fromCall(CallNode call, PointsToContext2 caller) { + caller.appliesTo(call) and + this = TCallContext(call, caller, _) + } + + /** Holds if `call` is the call-site from which this context was entered and `caller` is the caller's context. */ + predicate fromCall(CallNode call, PythonFunctionObjectInternal callee, PointsToContext2 caller) { + call = PointsTo2::get_a_call(callee, caller) and + this = TCallContext(call, caller, _) + } + + /** Gets the caller context for this callee context. */ + PointsToContext2 getOuter() { + this = TCallContext(_, result, _) + } + + /** Holds if this context is relevant to the given scope. */ + predicate appliesToScope(Scope s) { + /* Scripts */ + this = TMainContext() and maybe_main(s) + or + /* Modules and classes evaluated at import */ + s instanceof ImportTimeScope and this = TImportContext() + or + this = TRuntimeContext() and executes_in_runtime_context(s) + or + /* Called functions, regardless of their name */ + exists(PythonFunctionObjectInternal func, ControlFlowNode call, TPointsToContext outerContext | + call = PointsTo2::get_a_call(func, outerContext) and + this = TCallContext(call, outerContext, _) and + s = func.getScope() + ) + // TO DO... + //or + //exists(FunctionObject func | + // PointsTo2::Flow::callsite_calls_function(_, _, func, this, _) and + // s = func.getFunction() + //) + } + + /** Holds if this context can apply to the CFG node `n`. */ + pragma [inline] + predicate appliesTo(ControlFlowNode n) { + this.appliesToScope(n.getScope()) + } + + /** Holds if this context is a call context. */ + predicate isCall() { + this = TCallContext(_, _, _) + } + + /** Holds if this is the "main" context. */ + predicate isMain() { + this = TMainContext() + } + + /** Holds if this is the "import" context. */ + predicate isImport() { + this = TImportContext() + } + + /** Holds if this is the "default" context. */ + predicate isRuntime() { + this = TRuntimeContext() + } + + /** Holds if this context or one of its caller contexts is the default context. */ + predicate fromRuntime() { + this.isRuntime() + or + this.getOuter().fromRuntime() + } + + /** Gets the depth (number of calls) for this context. */ + int getDepth() { + not exists(this.getOuter()) and result = 0 + or + result = this.getOuter().getDepth() + 1 + } + + int getCost() { + result = context_cost(this) + } + + CallNode getCall() { + this = TCallContext(result, _, _) + } + + /** Holds if a call would be too expensive to create a new context for */ + predicate untrackableCall(CallNode call) { + total_cost(call, this) > max_context_cost() + } + + CallNode getRootCall() { + this = TCallContext(result, TImportContext(), _) + or + result = this.getOuter().getRootCall() + } + + /** Gets a version of Python that this context includes */ + pragma [inline] + Version getAVersion() { + /* Currently contexts do not include any version information, but may do in the future */ + result = major_version() + } + +} + +private predicate in_source(Scope s) { + exists(s.getEnclosingModule().getFile().getRelativePath()) +} + +/** Holds if this scope can be executed in the default context. + * All modules and classes executed at import time and + * all "public" functions and methods, including those invoked by the VM. + */ +predicate executes_in_runtime_context(Function f) { + /* "Public" scope, i.e. functions whose name starts not with an underscore, or special methods */ + (f.getName().charAt(0) != "_" or f.isSpecialMethod() or f.isInitMethod()) + and + in_source(f) +} + +private predicate maybe_main(Module m) { + exists(If i, Compare cmp, Name name, StrConst main | + m.getAStmt() = i and i.getTest() = cmp | + cmp.compares(name, any(Eq eq), main) and + name.getId() = "__name__" and + main.getText() = "__main__" + ) +} + diff --git a/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.expected b/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.expected new file mode 100644 index 000000000000..6de862cee891 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.expected @@ -0,0 +1,13 @@ +| test.py:7:1:7:1 | ControlFlowNode for n | None | test.py:3:5:3:8 | ControlFlowNode for None | +| test.py:8:1:8:1 | ControlFlowNode for t | True | test.py:4:5:4:8 | ControlFlowNode for True | +| test.py:9:1:9:1 | ControlFlowNode for f | False | test.py:5:5:5:9 | ControlFlowNode for False | +| test.py:14:5:14:5 | ControlFlowNode for f | False | test.py:5:5:5:9 | ControlFlowNode for False | +| test.py:22:5:22:5 | ControlFlowNode for n | None | test.py:18:9:18:12 | ControlFlowNode for None | +| test.py:23:5:23:5 | ControlFlowNode for t | True | test.py:19:9:19:12 | ControlFlowNode for True | +| test.py:24:5:24:5 | ControlFlowNode for f | False | test.py:20:9:20:13 | ControlFlowNode for False | +| test.py:29:9:29:9 | ControlFlowNode for f | False | test.py:20:9:20:13 | ControlFlowNode for False | +| test.py:31:5:31:9 | ControlFlowNode for UnaryExpr | True | test.py:31:5:31:9 | ControlFlowNode for UnaryExpr | +| test.py:32:5:32:9 | ControlFlowNode for UnaryExpr | False | test.py:32:5:32:9 | ControlFlowNode for UnaryExpr | +| test.py:33:5:33:9 | ControlFlowNode for UnaryExpr | True | test.py:33:5:33:9 | ControlFlowNode for UnaryExpr | +| test.py:38:9:38:9 | ControlFlowNode for f | False | test.py:20:9:20:13 | ControlFlowNode for False | +| test.py:41:9:41:9 | ControlFlowNode for t | True | test.py:19:9:19:12 | ControlFlowNode for True | diff --git a/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.ql b/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.ql new file mode 100644 index 000000000000..e30bd4b3f3a3 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.ql @@ -0,0 +1,10 @@ + +import python +import semmle.python.pointsto.PointsTo2 +import semmle.python.objects.TObject + +from ControlFlowNode f, ObjectInternal obj, ControlFlowNode orig +where exists(ExprStmt s | s.getValue().getAFlowNode() = f) and +PointsTo2::points_to(f, _, obj, orig) + +select f, obj.toString(), orig diff --git a/python/ql/test/library-tests/PointsTo/local/test.py b/python/ql/test/library-tests/PointsTo/local/test.py new file mode 100644 index 000000000000..c830d2dd33da --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/local/test.py @@ -0,0 +1,44 @@ +#Global + +n = None +t = True +f = False + +n +t +f + +if n: + t +else: + f + + +def foo(): + n = None + t = True + f = False + + n + t + f + + if n: + t + else: + f + + not n + not t + not f + + if t == f: + t + else: + f + + if t is not f: + t + else: + f + From 051683fadfbc044e5808d4ebec5608525ea9ba90 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 19 Mar 2019 15:07:12 +0000 Subject: [PATCH 002/128] Python: Break-up internal object modules. --- .../ql/src/semmle/python/objects/TObject.qll | 915 ------------------ .../src/semmle/python/pointsto/PointsTo2.qll | 2 +- .../python/pointsto/PointsToContext2.qll | 2 +- .../PointsTo/local/LocalPointsTo.ql | 2 +- 4 files changed, 3 insertions(+), 918 deletions(-) diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 812026ed8905..9e05a00e1f3e 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -1,7 +1,5 @@ import python private import semmle.python.types.Builtins -private import semmle.python.pointsto.PointsTo2 -private import semmle.python.pointsto.PointsToContext2 newtype TObject = TBuiltinClassObject(Builtin bltn) { @@ -66,916 +64,3 @@ library class ClassDecl extends @py_object { ) } } - -abstract class ObjectInternal extends TObject { - - abstract string toString(); - - /** The boolean value of this object, if it has one */ - abstract boolean booleanValue(); - - /** Holds if this object may be true or false when evaluated as a bool */ - abstract predicate maybe(); - - abstract predicate instantiated(ControlFlowNode node, PointsToContext2 context); - - /** Gets the class declaration for this object, if it is a declared class. */ - abstract ClassDecl getClassDeclaration(); - - abstract predicate isClass(); - - abstract predicate notClass(); - - abstract ObjectInternal getClass(); - - /** Holds if whatever this "object" represents can be meaningfully analysed for - * truth or false in comparisons. For example, `None` or `int` can be, but `int()` - * or an unknown string cannot. - */ - abstract predicate isComparable(); - - /** The negation of `isComparable()` */ - abstract predicate notComparable(); - - /** Gets the `Builtin` for this object, if any. - * All objects (except unknown and undefined values) should return - * exactly one result for either this method or `getOrigin()`. - */ - abstract Builtin getBuiltin(); - - /** Gets a control flow node that represents the source origin of this - * objects. - * All objects (except unknown and undefined values) should return - * exactly one result for either this method or `getBuiltin()`. - */ - abstract ControlFlowNode getOrigin(); - - /** Holds if `obj` is the result of calling `this` and `origin` is - * the origin of `obj`. - */ - abstract predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin); - - predicate hasLocationInfo(string fp, int bl, int bc, int el, int ec) { - this.getOrigin().getLocation().hasLocationInfo(fp, bl, bc, el, ec) - } - -} - -class PythonFunctionObjectInternal extends ObjectInternal, TPythonFunctionObject { - - Function getScope() { - exists(CallableExpr expr | - this = TPythonFunctionObject(expr.getAFlowNode()) and - result = expr.getInnerScope() - ) - } - - override string toString() { - result = this.getScope().toString() - } - - /** The boolean value of this object, if it has one */ - override boolean booleanValue() { - result = true - } - - /** Holds if this object may be true or false when evaluated as a bool */ - override predicate maybe() { none() } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - this = TPythonFunctionObject(node) and context.appliesTo(node) - } - - /** INTERNAL */ - override ClassDecl getClassDeclaration() { none() } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(Builtin::special("FunctionType")) - } - - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - - override Builtin getBuiltin() { - none() - } - - override ControlFlowNode getOrigin() { - this = TPythonFunctionObject(result) - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - exists(Function func, ControlFlowNode rval | - func = this.getScope() and - callee.appliesToScope(func) and - rval = func.getAReturnValueFlowNode() and - PointsTo2::points_to(rval, callee, obj, origin) - ) - } - -} - - -abstract class ClassObjectInternal extends ObjectInternal { - - override boolean booleanValue() { - result = true - } - - override predicate maybe() { none() } - - override predicate isClass() { any() } - - override predicate notClass() { none() } - - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - // TO DO .. Result should (in most cases) be an instance - none() - } - -} - -class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject { - - Class getScope() { - exists(ClassDef def | - this = TPythonClassObject(def.getAFlowNode()) and - result = def.getDefinedClass() - ) - } - - override string toString() { - result = this.getScope().toString() - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - exists(DefinitionNode def | - this = TPythonClassObject(def) and - node = def.getValue() and - context.appliesTo(node) - ) - } - - override ClassDecl getClassDeclaration() { - this = TPythonClassObject(result) - } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(Builtin::special("FunctionType")) - } - - override Builtin getBuiltin() { - none() - } - - override ControlFlowNode getOrigin() { - this = TPythonClassObject(result) - } - -} - -class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObject { - - override Builtin getBuiltin() { - this = TBuiltinClassObject(result) - } - - override string toString() { - result = "builtin class " + this.getBuiltin().getName() - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - none() - } - - override ClassDecl getClassDeclaration() { - this = TBuiltinClassObject(result) - } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(this.getBuiltin().getClass()) - } - - override ControlFlowNode getOrigin() { - none() - } - -} - -abstract class ModuleObjectInternal extends ObjectInternal { - - abstract string getName(); - - abstract Module getSourceModule(); - - abstract predicate isBuiltin(); - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - // Modules aren't callable - none() - } - - override ControlFlowNode getOrigin() { - result = this.getSourceModule().getEntryNode() - } - -} - -class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleObject { - - override Builtin getBuiltin() { - this = TBuiltinModuleObject(result) - } - - override string toString() { - result = "builtin module " + this.getBuiltin().getName() - } - - override string getName() { - result = this.getBuiltin().getName() - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - none() - } - - override boolean booleanValue() { - result = true - } - - override predicate maybe() { none() } - - override ClassDecl getClassDeclaration() { - none() - } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(this.getBuiltin().getClass()) - } - - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - - override Module getSourceModule() { - none() - } - - override predicate isBuiltin() { - any() - } - -} - -class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { - - override Builtin getBuiltin() { - none() - } - - Folder getFolder() { - this = TPackageObject(result) - } - - override string toString() { - result = "package " + this.getName() - } - - override string getName() { - result = moduleNameFromFile(this.getFolder()) - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - none() - } - - override boolean booleanValue() { - result = true - } - - override predicate maybe() { none() } - - override ClassDecl getClassDeclaration() { - none() - } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(this.getBuiltin().getClass()) - } - - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - - override Module getSourceModule() { - result.getFile() = this.getFolder().getFile("__init__.py") - } - - PythonModuleObjectInternal getInitModule() { - result = TPythonModule(this.getSourceModule()) - } - - predicate hasNoInitModule() { - exists(Folder f | - f = this.getFolder() and - not exists(f.getFile("__init__.py")) - ) - } - - override predicate isBuiltin() { - none() - } - - ModuleObjectInternal submodule(string name) { - result.getName() = this.getName() + "." + name - } - - override predicate hasLocationInfo(string fp, int bl, int bc, int el, int ec) { - this.getOrigin().getLocation().hasLocationInfo(fp, bl, bc, el, ec) - or - this.hasNoInitModule() and fp = this.getFolder().getAbsolutePath() and - bl = 0 and bc = 0 and el = 0 and ec = 0 - } - -} - -class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { - - override Builtin getBuiltin() { - none() - } - - override string toString() { - result = "package " + this.getName() - } - - override string getName() { - result = this.getSourceModule().getName() - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - none() - } - - override boolean booleanValue() { - result = true - } - - override predicate maybe() { none() } - - override ClassDecl getClassDeclaration() { - none() - } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(this.getBuiltin().getClass()) - } - - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - - override Module getSourceModule() { - this = TPythonModule(result) - } - - PythonModuleObjectInternal getInitModule() { - result = TPythonModule(this.getSourceModule()) - } - - override predicate isBuiltin() { - none() - } - -} - - -class BuiltinFunctionObjectInternal extends ObjectInternal, TBuiltinFunctionObject { - - override Builtin getBuiltin() { - this = TBuiltinFunctionObject(result) - } - - override string toString() { - result = "builtin function " + this.getBuiltin().getName() - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - none() - } - - override predicate maybe() { none() } - - override ClassDecl getClassDeclaration() { - none() - } - - override boolean booleanValue() { - result = true - } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(this.getBuiltin().getClass()) - } - - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. - none() - } - - override ControlFlowNode getOrigin() { - none() - } - -} - - -class BuiltinMethodObjectInternal extends ObjectInternal, TBuiltinMethodObject { - - override Builtin getBuiltin() { - this = TBuiltinMethodObject(result) - } - - override string toString() { - result = "builtin method " + this.getBuiltin().getName() - } - - override boolean booleanValue() { - result = true - } - - override predicate maybe() { none() } - - override ClassDecl getClassDeclaration() { - none() - } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(this.getBuiltin().getClass()) - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - none() - } - - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. - none() - } - - override ControlFlowNode getOrigin() { - none() - } - -} - - -class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { - - override Builtin getBuiltin() { - this = TBuiltinOpaqueObject(result) - } - - override string toString() { - none() - } - - override boolean booleanValue() { - // TO DO ... Depends on class. `this.getClass().instancesAlways(result)` - none() - } - - override predicate maybe() { - // TO DO ... Depends on class. `this.getClass().instancesMaybe()` - any() - } - - override ClassDecl getClassDeclaration() { - none() - } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(this.getBuiltin().getClass()) - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - none() - } - - override predicate isComparable() { - none() - } - - override predicate notComparable() { - any() - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and - callee_for_object(callee, this) - } - - override ControlFlowNode getOrigin() { - none() - } - -} - -private predicate callee_for_object(PointsToContext2 callee, ObjectInternal obj) { - exists(CallNode call, PointsToContext2 caller | - callee.fromCall(call, caller) and - PointsTo2::points_to(call.getFunction(), caller, obj, _) - ) -} - - -abstract class BooleanObjectInternal extends ObjectInternal { - - BooleanObjectInternal() { - this = TTrue() or this = TFalse() - } - - override predicate maybe() { none() } - - override ClassDecl getClassDeclaration() { - none() - } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(Builtin::special("bool")) - } - - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - - override Builtin getBuiltin() { - none() - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - // Booleans aren't callable - none() - } - - override ControlFlowNode getOrigin() { - none() - } - -} - -class TrueObjectInternal extends BooleanObjectInternal, TTrue { - - override string toString() { - result = "True" - } - - override boolean booleanValue() { - result = true - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - node.(NameNode).getId() = "True" and context.appliesTo(node) - } - -} - -class FalseObjectInternal extends BooleanObjectInternal, TFalse { - - override string toString() { - result = "False" - } - - override boolean booleanValue() { - result = false - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - node.(NameNode).getId() = "False" and context.appliesTo(node) - } - -} - -class NoneObjectInternal extends ObjectInternal, TNone { - - override string toString() { - result = "None" - } - - override boolean booleanValue() { - result = false - } - - override predicate maybe() { none() } - - override ClassDecl getClassDeclaration() { - none() - } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(Builtin::special("NoneType")) - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - node.(NameNode).getId() = "None" and context.appliesTo(node) - } - - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - - override Builtin getBuiltin() { - none() - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - // None isn't callable - none() - } - - override ControlFlowNode getOrigin() { - none() - } - -} - - -class UnknownInternal extends ObjectInternal, TUnknown { - - override string toString() { - none() - } - - override boolean booleanValue() { - none() - } - - override predicate maybe() { any() } - - override ClassDecl getClassDeclaration() { - none() - } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - result = TUnknownClass() - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - none() - } - - override predicate isComparable() { - none() - } - - override predicate notComparable() { - any() - } - - override Builtin getBuiltin() { - none() - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and - callee_for_object(callee, this) - } - - override ControlFlowNode getOrigin() { - none() - } - -} - -class UnknownClassInternal extends ObjectInternal, TUnknownClass { - - override string toString() { - none() - } - - override boolean booleanValue() { - none() - } - - override predicate maybe() { any() } - - override ClassDecl getClassDeclaration() { - result = Builtin::unknownType() - } - - override predicate isClass() { any() } - - override predicate notClass() { none() } - - override ObjectInternal getClass() { - result = TUnknownClass() - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - none() - } - - override predicate isComparable() { - none() - } - - override predicate notComparable() { - any() - } - - override Builtin getBuiltin() { - none() - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and - callee_for_object(callee, this) - } - - override ControlFlowNode getOrigin() { - none() - } - -} - -class UndefinedInternal extends ObjectInternal, TUndefined { - - override string toString() { - none() - } - - override boolean booleanValue() { - none() - } - - override predicate maybe() { none() } - - override ClassDecl getClassDeclaration() { - none() - } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - none() - } - - override predicate instantiated(ControlFlowNode node, PointsToContext2 context) { - none() - } - - override predicate isComparable() { - none() - } - - override predicate notComparable() { - any() - } - - override Builtin getBuiltin() { - none() - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - // Accessing an undefined value raises a NameError, but if during import it probably - // means that we missed an import. - obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and - callee.getOuter().isImport() - } - - override ControlFlowNode getOrigin() { - none() - } - -} - -module ObjectInternal { - - ObjectInternal bool(boolean b) { - b = true and result = TTrue() - or - b = false and result = TFalse() - } - - ObjectInternal none_() { - result = TNone() - } - - - ObjectInternal unknown() { - result = TUnknown() - } - - ObjectInternal unknownClass() { - result = TUnknownClass() - } - - ObjectInternal undefined() { - result = TUndefined() - } - - ObjectInternal builtin(string name) { - result = TBuiltinClassObject(Builtin::builtin(name)) - or - result = TBuiltinFunctionObject(Builtin::builtin(name)) - or - result = TBuiltinOpaqueObject(Builtin::builtin(name)) - } - - ObjectInternal sysModules() { - result = TBuiltinOpaqueObject(Builtin::special("sys").getMember("modules")) - } - -} - diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index d2e55dcc3b57..2f69af4b9a7a 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -1,6 +1,6 @@ import python -private import semmle.python.objects.TObject +private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.Filters private import semmle.python.pointsto.PointsToContext2 diff --git a/python/ql/src/semmle/python/pointsto/PointsToContext2.qll b/python/ql/src/semmle/python/pointsto/PointsToContext2.qll index 622213c40ecf..9e09cb02bb9f 100644 --- a/python/ql/src/semmle/python/pointsto/PointsToContext2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsToContext2.qll @@ -1,6 +1,6 @@ import python private import semmle.python.pointsto.PointsTo2 -private import semmle.python.objects.TObject +private import semmle.python.objects.ObjectInternal /* * A note on 'cost'. Cost doesn't represent the cost to compute, * but (a vague estimate of) the cost to compute per value gained. diff --git a/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.ql b/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.ql index e30bd4b3f3a3..767736718626 100644 --- a/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.ql +++ b/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.ql @@ -1,7 +1,7 @@ import python import semmle.python.pointsto.PointsTo2 -import semmle.python.objects.TObject +import semmle.python.objects.ObjectInternal from ControlFlowNode f, ObjectInternal obj, ControlFlowNode orig where exists(ExprStmt s | s.getValue().getAFlowNode() = f) and From 5a46df2132d033165900c74b985a0a28d41d041a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 19 Mar 2019 16:26:55 +0000 Subject: [PATCH 003/128] Python: Add ADTs for ints and strings. Add some global data-flow. --- .../ql/src/semmle/python/objects/Classes.qll | 111 ++++ .../src/semmle/python/objects/Constants.qll | 256 +++++++++ .../ql/src/semmle/python/objects/Modules.qll | 219 ++++++++ .../semmle/python/objects/ObjectInternal.qll | 518 ++++++++++++++++++ .../ql/src/semmle/python/objects/TObject.qll | 31 ++ .../src/semmle/python/pointsto/PointsTo2.qll | 278 +++++++--- .../python/pointsto/PointsToContext2.qll | 2 +- .../PointsTo/global/Global.expected | 30 + .../library-tests/PointsTo/global/Global.ql | 14 + .../library-tests/PointsTo/global/test.py | 22 + .../PointsTo/local/LocalPointsTo.expected | 9 + .../test/library-tests/PointsTo/local/test.py | 33 +- 12 files changed, 1452 insertions(+), 71 deletions(-) create mode 100644 python/ql/src/semmle/python/objects/Classes.qll create mode 100644 python/ql/src/semmle/python/objects/Constants.qll create mode 100644 python/ql/src/semmle/python/objects/Modules.qll create mode 100644 python/ql/src/semmle/python/objects/ObjectInternal.qll create mode 100644 python/ql/test/library-tests/PointsTo/global/Global.expected create mode 100644 python/ql/test/library-tests/PointsTo/global/Global.ql create mode 100644 python/ql/test/library-tests/PointsTo/global/test.py diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll new file mode 100644 index 000000000000..b82321d14106 --- /dev/null +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -0,0 +1,111 @@ +import python + + +private import semmle.python.objects.TObject +private import semmle.python.objects.ObjectInternal +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.types.Builtins + + +abstract class ClassObjectInternal extends ObjectInternal { + + override boolean booleanValue() { + result = true + } + + override predicate maybe() { none() } + + override predicate isClass() { any() } + + override predicate notClass() { none() } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // TO DO .. Result should (in most cases) be an instance + none() + } + +} + +class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject { + + Class getScope() { + exists(ClassDef def | + this = TPythonClassObject(def.getAFlowNode()) and + result = def.getDefinedClass() + ) + } + + override string toString() { + result = this.getScope().toString() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + exists(DefinitionNode def | + this = TPythonClassObject(def) and + node = def.getValue() and + context.appliesTo(node) + ) + } + + override ClassDecl getClassDeclaration() { + this = TPythonClassObject(result) + } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("FunctionType")) + } + + override Builtin getBuiltin() { + none() + } + + override ControlFlowNode getOrigin() { + this = TPythonClassObject(result) + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + exists(PythonFunctionObjectInternal init | + // TO DO... Lookup init... + none() | + init.getScope() = scope and paramOffset = 1 + ) + } + +} + +class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObject { + + override Builtin getBuiltin() { + this = TBuiltinClassObject(result) + } + + override string toString() { + result = "builtin class " + this.getBuiltin().getName() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override ClassDecl getClassDeclaration() { + this = TBuiltinClassObject(result) + } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override ControlFlowNode getOrigin() { + none() + } + +} diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll new file mode 100644 index 000000000000..abb419203857 --- /dev/null +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -0,0 +1,256 @@ +import python + +private import semmle.python.objects.TObject +private import semmle.python.objects.ObjectInternal +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.types.Builtins + + +abstract class BooleanObjectInternal extends ObjectInternal { + + BooleanObjectInternal() { + this = TTrue() or this = TFalse() + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("bool")) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Builtin getBuiltin() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // Booleans aren't callable + none() + } + + override ControlFlowNode getOrigin() { + none() + } + +} + +class TrueObjectInternal extends BooleanObjectInternal, TTrue { + + override string toString() { + result = "True" + } + + override boolean booleanValue() { + result = true + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + node.(NameNode).getId() = "True" and context.appliesTo(node) + } + +} + +class FalseObjectInternal extends BooleanObjectInternal, TFalse { + + override string toString() { + result = "False" + } + + override boolean booleanValue() { + result = false + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + node.(NameNode).getId() = "False" and context.appliesTo(node) + } + +} + +class NoneObjectInternal extends ObjectInternal, TNone { + + override string toString() { + result = "None" + } + + override boolean booleanValue() { + result = false + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("NoneType")) + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + node.(NameNode).getId() = "None" and context.appliesTo(node) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Builtin getBuiltin() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // None isn't callable + none() + } + + override ControlFlowNode getOrigin() { + none() + } + +} + + +class IntObjectInternal extends ObjectInternal, TInt { + + override string toString() { + result = this.intValue().toString() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + context.appliesTo(node) and + node.getNode().(IntegerLiteral).getValue() = this.intValue() + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("int")) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Builtin getBuiltin() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // ints aren't callable + none() + } + + override ControlFlowNode getOrigin() { + none() + } + + override int intValue() { + this = TInt(result) + } + + override boolean booleanValue() { + this.intValue() = 0 and result = false + or + this.intValue() != 0 and result = true + } + +} + + +class StringObjectInternal extends ObjectInternal, TString { + + override string toString() { + result = "'" + this.strValue().toString() + "'" + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + context.appliesTo(node) and + node.getNode().(StrConst).getText() = this.strValue() + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("str")) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Builtin getBuiltin() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // strings aren't callable + none() + } + + override ControlFlowNode getOrigin() { + none() + } + + override string strValue() { + this = TString(result) + } + + override boolean booleanValue() { + this.strValue() = "" and result = false + or + this.strValue() != "" and result = true + } + +} + + + + diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll new file mode 100644 index 000000000000..d2d325a38858 --- /dev/null +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -0,0 +1,219 @@ +import python + +private import semmle.python.objects.TObject +private import semmle.python.objects.ObjectInternal +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.types.Builtins + +abstract class ModuleObjectInternal extends ObjectInternal { + + abstract string getName(); + + abstract Module getSourceModule(); + + abstract predicate isBuiltin(); + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // Modules aren't callable + none() + } + + override ControlFlowNode getOrigin() { + result = this.getSourceModule().getEntryNode() + } + +} + +class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleObject { + + override Builtin getBuiltin() { + this = TBuiltinModuleObject(result) + } + + override string toString() { + result = "builtin module " + this.getBuiltin().getName() + } + + override string getName() { + result = this.getBuiltin().getName() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override boolean booleanValue() { + result = true + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Module getSourceModule() { + none() + } + + override predicate isBuiltin() { + any() + } + +} + +class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { + + override Builtin getBuiltin() { + none() + } + + Folder getFolder() { + this = TPackageObject(result) + } + + override string toString() { + result = "package " + this.getName() + } + + override string getName() { + result = moduleNameFromFile(this.getFolder()) + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override boolean booleanValue() { + result = true + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Module getSourceModule() { + result.getFile() = this.getFolder().getFile("__init__.py") + } + + PythonModuleObjectInternal getInitModule() { + result = TPythonModule(this.getSourceModule()) + } + + predicate hasNoInitModule() { + exists(Folder f | + f = this.getFolder() and + not exists(f.getFile("__init__.py")) + ) + } + + override predicate isBuiltin() { + none() + } + + ModuleObjectInternal submodule(string name) { + result.getName() = this.getName() + "." + name + } + + override predicate hasLocationInfo(string fp, int bl, int bc, int el, int ec) { + this.getOrigin().getLocation().hasLocationInfo(fp, bl, bc, el, ec) + or + this.hasNoInitModule() and fp = this.getFolder().getAbsolutePath() and + bl = 0 and bc = 0 and el = 0 and ec = 0 + } + +} + +class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { + + override Builtin getBuiltin() { + none() + } + + override string toString() { + result = "package " + this.getName() + } + + override string getName() { + result = this.getSourceModule().getName() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override boolean booleanValue() { + result = true + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Module getSourceModule() { + this = TPythonModule(result) + } + + PythonModuleObjectInternal getInitModule() { + result = TPythonModule(this.getSourceModule()) + } + + override predicate isBuiltin() { + none() + } + +} + diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll new file mode 100644 index 000000000000..8e9d13507df5 --- /dev/null +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -0,0 +1,518 @@ +import python + +private import semmle.python.objects.TObject +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.types.Builtins +import semmle.python.objects.Modules +import semmle.python.objects.Classes +import semmle.python.objects.Constants + +abstract class ObjectInternal extends TObject { + + abstract string toString(); + + /** The boolean value of this object, if it has one */ + abstract boolean booleanValue(); + + /** Holds if this object may be true or false when evaluated as a bool */ + abstract predicate maybe(); + + abstract predicate introduced(ControlFlowNode node, PointsToContext2 context); + + /** Gets the class declaration for this object, if it is a declared class. */ + abstract ClassDecl getClassDeclaration(); + + abstract predicate isClass(); + + abstract predicate notClass(); + + abstract ObjectInternal getClass(); + + /** Holds if whatever this "object" represents can be meaningfully analysed for + * truth or false in comparisons. For example, `None` or `int` can be, but `int()` + * or an unknown string cannot. + */ + abstract predicate isComparable(); + + /** The negation of `isComparable()` */ + abstract predicate notComparable(); + + /** Gets the `Builtin` for this object, if any. + * All objects (except unknown and undefined values) should return + * exactly one result for either this method or `getOrigin()`. + */ + abstract Builtin getBuiltin(); + + /** Gets a control flow node that represents the source origin of this + * objects. + * All objects (except unknown and undefined values) should return + * exactly one result for either this method or `getBuiltin()`. + */ + abstract ControlFlowNode getOrigin(); + + /** Holds if `obj` is the result of calling `this` and `origin` is + * the origin of `obj`. + */ + abstract predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin); + + predicate hasLocationInfo(string fp, int bl, int bc, int el, int ec) { + this.getOrigin().getLocation().hasLocationInfo(fp, bl, bc, el, ec) + } + + /** The integer value of things that have integer values. + * That is, ints and bools. + */ + int intValue() { + none() + } + + /** The integer value of things that have integer values. + * That is, strings. + */ + string strValue() { + none() + } + + predicate calleeAndOffset(Function scope, int paramOffset) { none() } + +} + + +class PythonFunctionObjectInternal extends ObjectInternal, TPythonFunctionObject { + + Function getScope() { + exists(CallableExpr expr | + this = TPythonFunctionObject(expr.getAFlowNode()) and + result = expr.getInnerScope() + ) + } + + override string toString() { + result = this.getScope().toString() + } + + /** The boolean value of this object, if it has one */ + override boolean booleanValue() { + result = true + } + + /** Holds if this object may be true or false when evaluated as a bool */ + override predicate maybe() { none() } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + this = TPythonFunctionObject(node) and context.appliesTo(node) + } + + /** INTERNAL */ + override ClassDecl getClassDeclaration() { none() } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("FunctionType")) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Builtin getBuiltin() { + none() + } + + override ControlFlowNode getOrigin() { + this = TPythonFunctionObject(result) + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + exists(Function func, ControlFlowNode rval | + func = this.getScope() and + callee.appliesToScope(func) and + rval = func.getAReturnValueFlowNode() and + PointsTo2::points_to(rval, callee, obj, origin) + ) + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + scope = this.getScope() and paramOffset = 0 + } + +} + +/// BOUND METHODS +/// +/// +/// + +class BuiltinFunctionObjectInternal extends ObjectInternal, TBuiltinFunctionObject { + + override Builtin getBuiltin() { + this = TBuiltinFunctionObject(result) + } + + override string toString() { + result = "builtin function " + this.getBuiltin().getName() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override boolean booleanValue() { + result = true + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. + none() + } + + override ControlFlowNode getOrigin() { + none() + } + +} + + +class BuiltinMethodObjectInternal extends ObjectInternal, TBuiltinMethodObject { + + override Builtin getBuiltin() { + this = TBuiltinMethodObject(result) + } + + override string toString() { + result = "builtin method " + this.getBuiltin().getName() + } + + override boolean booleanValue() { + result = true + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. + none() + } + + override ControlFlowNode getOrigin() { + none() + } + +} + + +class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { + + override Builtin getBuiltin() { + this = TBuiltinOpaqueObject(result) + } + + override string toString() { + none() + } + + override boolean booleanValue() { + // TO DO ... Depends on class. `this.getClass().instancesAlways(result)` + none() + } + + override predicate maybe() { + // TO DO ... Depends on class. `this.getClass().instancesMaybe()` + any() + } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate isComparable() { + none() + } + + override predicate notComparable() { + any() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and + callee_for_object(callee, this) + } + + override ControlFlowNode getOrigin() { + none() + } + +} + +private predicate callee_for_object(PointsToContext2 callee, ObjectInternal obj) { + exists(CallNode call, PointsToContext2 caller | + callee.fromCall(call, caller) and + PointsTo2::points_to(call.getFunction(), caller, obj, _) + ) +} + + +class UnknownClassInternal extends ObjectInternal, TUnknownClass { + + override string toString() { + none() + } + + override boolean booleanValue() { + none() + } + + override predicate maybe() { any() } + + override ClassDecl getClassDeclaration() { + result = Builtin::unknownType() + } + + override predicate isClass() { any() } + + override predicate notClass() { none() } + + override ObjectInternal getClass() { + result = TUnknownClass() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate isComparable() { + none() + } + + override predicate notComparable() { + any() + } + + override Builtin getBuiltin() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and + callee_for_object(callee, this) + } + + override ControlFlowNode getOrigin() { + none() + } + +} + +class UnknownInternal extends ObjectInternal, TUnknown { + + override string toString() { + none() + } + + override boolean booleanValue() { + none() + } + + override predicate maybe() { any() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + result = TUnknownClass() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate isComparable() { + none() + } + + override predicate notComparable() { + any() + } + + override Builtin getBuiltin() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and + callee_for_object(callee, this) + } + + override ControlFlowNode getOrigin() { + none() + } + +} + +class UndefinedInternal extends ObjectInternal, TUndefined { + + override string toString() { + none() + } + + override boolean booleanValue() { + none() + } + + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + override ObjectInternal getClass() { + none() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate isComparable() { + none() + } + + override predicate notComparable() { + any() + } + + override Builtin getBuiltin() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // Accessing an undefined value raises a NameError, but if during import it probably + // means that we missed an import. + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and + callee.getOuter().isImport() + } + + override ControlFlowNode getOrigin() { + none() + } + +} + +module ObjectInternal { + + ObjectInternal bool(boolean b) { + b = true and result = TTrue() + or + b = false and result = TFalse() + } + + ObjectInternal none_() { + result = TNone() + } + + + ObjectInternal unknown() { + result = TUnknown() + } + + ObjectInternal unknownClass() { + result = TUnknownClass() + } + + ObjectInternal undefined() { + result = TUndefined() + } + + ObjectInternal builtin(string name) { + result = TBuiltinClassObject(Builtin::builtin(name)) + or + result = TBuiltinFunctionObject(Builtin::builtin(name)) + or + result = TBuiltinOpaqueObject(Builtin::builtin(name)) + } + + ObjectInternal sysModules() { + result = TBuiltinOpaqueObject(Builtin::special("sys").getMember("modules")) + } + + ObjectInternal fromInt(int n) { + result = TInt(n) + } + +} + diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 9e05a00e1f3e..fe2d60cce76f 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -41,8 +41,39 @@ newtype TObject = TUnknownClass() or TUndefined() + or + TInt(int n) { + // Powers of 2 are used for flags + is_power_2(n) or + // And all combinations of flags up to 2^8 + n in [0..511] or + // Any number explicitly mentioned in the source code. + exists(Num num | + n = num.getN().toInt() or + exists(UnaryExpr neg | neg.getOp() instanceof USub and neg.getOperand() = num) + and n = -num.getN().toInt() + ) + } + or + TString(string s) { + // Any string explicitly mentioned in the source code. + exists(StrConst str | + s = str.getText() + ) + or + // Any string from the library put in the DB by the extractor. + exists(string quoted_string, Builtin bltn | + quoted_string = bltn.getName() and + s = quoted_string.regexpCapture("[bu]'([\\s\\S]*)'", 1) + ) + } +private predicate is_power_2(int n) { + n = 1 or + exists(int half | is_power_2(half) and n = half*2) +} + library class ClassDecl extends @py_object { ClassDecl() { diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index 2f69af4b9a7a..7ae6b01e756f 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -101,9 +101,9 @@ module PointsTo2 { or origin = f and compare_expr_points_to(f, context, value) or - origin = f and not_points_to(f, context, value) + origin = f and unary_points_to(f, context, value) or - origin = f and value.instantiated(f, context) + origin = f and value.introduced(f, context) or InterModulePointsTo::import_points_to(f, context, value, origin) or @@ -255,7 +255,7 @@ module PointsTo2 { pragma [nomagic] private predicate ssa_node_definition_points_to_unpruned(EssaNodeDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { - parameter_points_to(def, context, value, origin) + InterProceduralPointsTo::parameter_points_to(def, context, value, origin) or assignment_points_to(def, context, value, origin) //// TO DO... @@ -320,32 +320,6 @@ module PointsTo2 { } - /** Points-to for parameter. `def foo(param): ...`. */ - pragma [noinline] - private predicate parameter_points_to(ParameterDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { - positional_parameter_points_to(def, context, value, origin) - //// TO DO... - // or - // named_parameter_points_to(def, context, value, origin) - // or - // default_parameter_points_to(def, context, value, origin) - // or - // special_parameter_points_to(def, context, value, origin) - } - - /** Helper for `parameter_points_to` */ - pragma [noinline] - private predicate positional_parameter_points_to(ParameterDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { - /// To do... - //exists(PointsToContext2 caller, ControlFlowNode arg | - // points_to(arg, caller, value, origin) and - // Flow::callsite_argument_transfer(arg, caller, def, context) - //) - //or - not def.isSelf() and not def.getParameter().isVarargs() and not def.getParameter().isKwargs() and - context.isRuntime() and value = ObjectInternal::unknown() and origin = def.getDefiningNode() - } - /** Holds if the phi-function `phi` refers to `(value, origin)` given the context `context`. */ pragma [nomagic] private predicate ssa_phi_points_to(PhiFunction phi, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { @@ -365,13 +339,12 @@ module PointsTo2 { pragma [noinline] private predicate scope_entry_points_to(ScopeEntryDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { /* Transfer from another scope */ - /// To do... - //exists(EssaVariable var, PointsToContext2 outer, CfgOrigin orig | - // Flow::scope_entry_value_transfer(var, outer, def, context) and - // ssa_variable_points_to(var, outer, value, orig) and - // origin = orig.asCfgNodeOrHere(def.getDefiningNode()) - //) - //or + exists(EssaVariable var, PointsToContext2 outer, CfgOrigin orig | + InterProceduralPointsTo::scope_entry_value_transfer(var, outer, def, context) and + ssa_variable_points_to(var, outer, value, orig) and + origin = orig.asCfgNodeOrHere(def.getDefiningNode()) + ) + or /* Undefined variable */ exists(Scope scope | not def.getVariable().getName() = "__name__" and @@ -411,48 +384,68 @@ module PointsTo2 { private predicate compare_expr_points_to(CompareNode cmp, PointsToContext2 context, ObjectInternal value) { exists(ControlFlowNode a, ControlFlowNode b, ObjectInternal o1, ObjectInternal o2 | exists(boolean is | - equality_test(cmp, a, is, b) and - points_to(a, context, o1, _) and - points_to(b, context, o2, _) | - (o1.isComparable() and o2.isComparable()) and - ( - o1 = o2 and value = ObjectInternal::bool(is) - or - o1 != o2 and value = ObjectInternal::bool(is.booleanNot()) - ) - or - (o1.notComparable() or o2.notComparable()) and - value = ObjectInternal::bool(_) - ) - // TO DO -- Comparison of int and string constants. Version tests and the like. - //or - //const_compare(cmp, context, comp, strict) - // exists(int comp, boolean strict | - // const_compare(cmp, context, comp, strict) - // | - // comp = -1 and value = theTrueObject() - // or - // comp = 0 and strict = false and value = ObjectInternal::bool(true) - // or - // comp = 0 and strict = true and value = ObjectInternal::bool(false) - // or - // comp = 1 and value = ObjectInternal::bool(false) - // ) + equality_test(cmp, a, is, b) and + points_to(a, context, o1, _) and + points_to(b, context, o2, _) | + (o1.isComparable() and o2.isComparable()) and + ( + o1 = o2 and value = ObjectInternal::bool(is) + or + o1 != o2 and value = ObjectInternal::bool(is.booleanNot()) + ) + or + (o1.notComparable() or o2.notComparable()) and + value = ObjectInternal::bool(_) + ) + or + exists(boolean strict | + inequality(cmp, a, b, strict) and + points_to(a, context, o1, _) and + points_to(b, context, o2, _) | + o1.intValue() < o2.intValue() and value = ObjectInternal::bool(true) + or + o1.intValue() > o2.intValue() and value = ObjectInternal::bool(false) + or + o1.intValue() = o2.intValue() and value = ObjectInternal::bool(strict.booleanNot()) + or + o1.strValue() < o2.strValue() and value = ObjectInternal::bool(true) + or + o1.strValue() > o2.strValue() and value = ObjectInternal::bool(false) + or + o1.strValue() = o2.strValue() and value = ObjectInternal::bool(strict.booleanNot()) + ) // or // value = version_tuple_compare(cmp, context) ) } - private predicate not_points_to(UnaryExprNode f, PointsToContext2 context, ObjectInternal value) { - f.getNode().getOp() instanceof Not and - exists(ObjectInternal operand | + /** Helper for comparisons. */ + predicate inequality(CompareNode cmp, ControlFlowNode lesser, ControlFlowNode greater, boolean strict) { + exists(Cmpop op | + cmp.operands(lesser, op, greater) and op.getSymbol() = "<" and strict = true + or + cmp.operands(lesser, op, greater) and op.getSymbol() = "<=" and strict = false + or + cmp.operands(greater, op, lesser) and op.getSymbol() = ">" and strict = true + or + cmp.operands(greater, op, lesser) and op.getSymbol() = ">=" and strict = false + ) + } + + private predicate unary_points_to(UnaryExprNode f, PointsToContext2 context, ObjectInternal value) { + exists(Unaryop op, ObjectInternal operand | + op = f.getNode().getOp() and points_to(f.getOperand(), context, operand, _) | - operand.maybe() and value = ObjectInternal::bool(_) + op instanceof Not and operand.maybe() and value = ObjectInternal::bool(_) or - operand.booleanValue() = false and value = ObjectInternal::bool(true) + op instanceof Not and operand.booleanValue() = false and value = ObjectInternal::bool(true) or - operand.booleanValue() = true and value = ObjectInternal::bool(false) + op instanceof Not and operand.booleanValue() = true and value = ObjectInternal::bool(false) + or + op instanceof USub and value = ObjectInternal::fromInt(-operand.intValue()) + or + operand = ObjectInternal::unknown() and value = operand ) } @@ -589,6 +582,122 @@ module InterProceduralPointsTo { ) } + /** Points-to for parameter. `def foo(param): ...`. */ + pragma [noinline] + predicate parameter_points_to(ParameterDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + positional_parameter_points_to(def, context, value, origin) + or + named_parameter_points_to(def, context, value, origin) + or + default_parameter_points_to(def, context, value, origin) + // or + // special_parameter_points_to(def, context, value, origin) + } + + /** Helper for `parameter_points_to` */ + pragma [noinline] + private predicate positional_parameter_points_to(ParameterDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + exists(PointsToContext2 caller, ControlFlowNode arg | + PointsTo2::points_to(arg, caller, value, origin) and + callsite_argument_transfer(arg, caller, def, context) + ) + or + not def.isSelf() and not def.getParameter().isVarargs() and not def.getParameter().isKwargs() and + context.isRuntime() and value = ObjectInternal::unknown() and origin = def.getDefiningNode() + } + + + /** Helper for `parameter_points_to` */ + pragma [noinline] + private predicate named_parameter_points_to(ParameterDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + exists(CallNode call, PointsToContext2 caller, PythonFunctionObjectInternal func, string name | + context.fromCall(call, func, caller) and + def.getParameter() = func.getScope().getArgByName(name) and + PointsTo2::points_to(call.getArgByName(name), caller, value, origin) + ) + } + + /** Helper for parameter_points_to */ + private predicate default_parameter_points_to(ParameterDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + exists(PointsToContext2 imp | imp.isImport() | PointsTo2::points_to(def.getDefault(), imp, value, origin)) and + context_for_default_value(def, context) + } + + /** Helper for default_parameter_points_to */ + pragma [noinline] + private predicate context_for_default_value(ParameterDefinition def, PointsToContext2 context) { + context.isRuntime() + or + exists(PointsToContext2 caller, CallNode call, PythonFunctionObjectInternal func, int n | + context.fromCall(call, func, caller) and + func.getScope().getArg(n) = def.getParameter() and + not exists(call.getArg(n)) and + not exists(call.getArgByName(def.getParameter().asName().getId())) and + not exists(call.getNode().getKwargs()) and + not exists(call.getNode().getStarargs()) + ) + } + + /** Holds if the `(argument, caller)` pair matches up with `(param, callee)` pair across call. */ + cached predicate callsite_argument_transfer(ControlFlowNode argument, PointsToContext2 caller, ParameterDefinition param, PointsToContext2 callee) { + exists(CallNode call, Function func, int n, int offset | + callsite_calls_function(call, caller, func, callee, offset) and + argument = call.getArg(n) and + param.getParameter() = func.getArg(n+offset) + ) + } + + cached predicate callsite_calls_function(CallNode call, PointsToContext2 caller, Function scope, PointsToContext2 callee, int parameter_offset) { + callee.fromCall(call, caller) and + exists(ObjectInternal func | + PointsTo2::points_to(call.getFunction(), caller, func, _) and + func.calleeAndOffset(scope, parameter_offset) + ) + } + + /** Model the transfer of values at scope-entry points. Transfer from `(pred_var, pred_context)` to `(succ_def, succ_context)`. */ + cached predicate scope_entry_value_transfer(EssaVariable pred_var, PointsToContext2 pred_context, ScopeEntryDefinition succ_def, PointsToContext2 succ_context) { + scope_entry_value_transfer_from_earlier(pred_var, pred_context, succ_def, succ_context) + // TO DO... + //or + //callsite_entry_value_transfer(pred_var, pred_context, succ_def, succ_context) + //or + //pred_context.isImport() and pred_context = succ_context and + //class_entry_value_transfer(pred_var, succ_def) + } + + /** Helper for `scope_entry_value_transfer`. Transfer of values from a temporally earlier scope to later scope. + * Earlier and later scopes are, for example, a module and functions in that module, or an __init__ method and another method. */ + pragma [noinline] + private predicate scope_entry_value_transfer_from_earlier(EssaVariable pred_var, PointsToContext2 pred_context, ScopeEntryDefinition succ_def, PointsToContext2 succ_context) { + exists(Scope pred_scope, Scope succ_scope | + BaseFlow::scope_entry_value_transfer_from_earlier(pred_var, pred_scope, succ_def, succ_scope) and + succ_context.appliesToScope(succ_scope) + | + succ_context.isRuntime() and succ_context = pred_context + or + pred_context.isImport() and pred_scope instanceof ImportTimeScope and + (succ_context.fromRuntime() or + /* A call made at import time, but from another module. Assume this module has been fully imported. */ + succ_context.isCall() and exists(CallNode call | succ_context.fromCall(call, _) and call.getEnclosingModule() != pred_scope)) + or + /* If predecessor scope is main, then we assume that any global defined exactly once + * is available to all functions. Although not strictly true, this gives less surprising + * results in practice. */ + pred_context.isMain() and pred_scope instanceof Module and succ_context.fromRuntime() and + exists(Variable v | + v = pred_var.getSourceVariable() and + not strictcount(v.getAStore()) > 1 + ) + ) + or + exists(NonEscapingGlobalVariable var | + var = pred_var.getSourceVariable() and var = succ_def.getSourceVariable() and + pred_var.getAUse() = succ_context.getRootCall() and pred_context.isImport() and + succ_context.appliesToScope(succ_def.getScope()) + ) + } + } /** Gets the `value, origin` that `f` would refer to if it has not been assigned some other value */ @@ -630,6 +739,8 @@ module Conditionals { //or result = equalityEvaluatesTo(expr, use, context, val, origin) or + result = inequalityEvaluatesTo(expr, use, context, val, origin) + or //result = callable_test_evaluates_boolean(expr, use, context, val, origin) //or //result = hasattr_test_evaluates_boolean(expr, use, context, val, origin) @@ -677,4 +788,33 @@ module Conditionals { ) } + pragma [noinline] + private boolean inequalityEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { + exists(ControlFlowNode r, boolean sense | + contains_interesting_expression_within_test(expr, use) | + exists(boolean strict, ObjectInternal other | + ( + PointsTo2::inequality(expr, use, r, strict) and sense = true + or + PointsTo2::inequality(expr, r, use, strict) and sense = false + ) and + PointsTo2::points_to(use, context, val, origin) and + PointsTo2::points_to(r, context, other, _) + | + val.intValue() < other.intValue() and result = sense + or + val.intValue() > other.intValue() and result = sense.booleanNot() + or + val.intValue() = other.intValue() and result = strict.booleanXor(sense) + or + val.strValue() < other.strValue() and result = sense + or + val.strValue() > other.strValue() and result = sense.booleanNot() + or + val.strValue() = other.strValue() and result = strict.booleanXor(sense) + ) + + ) + } + } diff --git a/python/ql/src/semmle/python/pointsto/PointsToContext2.qll b/python/ql/src/semmle/python/pointsto/PointsToContext2.qll index 9e09cb02bb9f..2e3cd1d80a7b 100644 --- a/python/ql/src/semmle/python/pointsto/PointsToContext2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsToContext2.qll @@ -8,7 +8,7 @@ private import semmle.python.objects.ObjectInternal */ private int given_cost() { - exists(string depth | + exists(string depth | py_flags_versioned("context.cost", depth, _) and result = depth.toInt() ) diff --git a/python/ql/test/library-tests/PointsTo/global/Global.expected b/python/ql/test/library-tests/PointsTo/global/Global.expected new file mode 100644 index 000000000000..f119830c37e9 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/global/Global.expected @@ -0,0 +1,30 @@ +| import | test.py:17:1:17:3 | ControlFlowNode for bar | Function bar | test.py:2:1:2:19 | ControlFlowNode for FunctionExpr | +| import | test.py:18:1:18:7 | ControlFlowNode for bar() | 21 | test.py:18:5:18:6 | ControlFlowNode for IntegerLiteral | +| import | test.py:19:1:19:13 | ControlFlowNode for bar() | 22 | test.py:19:5:19:6 | ControlFlowNode for IntegerLiteral | +| import | test.py:20:1:20:13 | ControlFlowNode for bar() | True | test.py:20:9:20:12 | ControlFlowNode for True | +| import | test.py:21:1:21:11 | ControlFlowNode for bar() | 24 | test.py:21:5:21:6 | ControlFlowNode for IntegerLiteral | +| import | test.py:22:1:22:13 | ControlFlowNode for bar() | 7 | test.py:22:7:22:7 | ControlFlowNode for IntegerLiteral | +| runtime | test.py:4:5:4:5 | ControlFlowNode for b | None | test.py:2:14:2:17 | ControlFlowNode for None | +| runtime | test.py:11:5:11:7 | ControlFlowNode for bar | Function bar | test.py:2:1:2:19 | ControlFlowNode for FunctionExpr | +| runtime | test.py:12:5:12:11 | ControlFlowNode for bar() | 11 | test.py:12:9:12:10 | ControlFlowNode for IntegerLiteral | +| runtime | test.py:13:5:13:17 | ControlFlowNode for bar() | 12 | test.py:13:9:13:10 | ControlFlowNode for IntegerLiteral | +| runtime | test.py:14:5:14:17 | ControlFlowNode for bar() | True | test.py:14:13:14:16 | ControlFlowNode for True | +| runtime | test.py:15:5:15:15 | ControlFlowNode for bar() | 14 | test.py:15:9:15:10 | ControlFlowNode for IntegerLiteral | +| test.py:12 from runtime | test.py:3:5:3:5 | ControlFlowNode for a | 11 | test.py:12:9:12:10 | ControlFlowNode for IntegerLiteral | +| test.py:12 from runtime | test.py:4:5:4:5 | ControlFlowNode for b | None | test.py:2:14:2:17 | ControlFlowNode for None | +| test.py:13 from runtime | test.py:3:5:3:5 | ControlFlowNode for a | 12 | test.py:13:9:13:10 | ControlFlowNode for IntegerLiteral | +| test.py:13 from runtime | test.py:4:5:4:5 | ControlFlowNode for b | None | test.py:13:13:13:16 | ControlFlowNode for None | +| test.py:14 from runtime | test.py:3:5:3:5 | ControlFlowNode for a | 13 | test.py:14:9:14:10 | ControlFlowNode for IntegerLiteral | +| test.py:14 from runtime | test.py:4:5:4:5 | ControlFlowNode for b | True | test.py:14:13:14:16 | ControlFlowNode for True | +| test.py:15 from runtime | test.py:3:5:3:5 | ControlFlowNode for a | 14 | test.py:15:9:15:10 | ControlFlowNode for IntegerLiteral | +| test.py:15 from runtime | test.py:4:5:4:5 | ControlFlowNode for b | '' | test.py:15:13:15:14 | ControlFlowNode for Str | +| test.py:18 from import | test.py:3:5:3:5 | ControlFlowNode for a | 21 | test.py:18:5:18:6 | ControlFlowNode for IntegerLiteral | +| test.py:18 from import | test.py:4:5:4:5 | ControlFlowNode for b | None | test.py:2:14:2:17 | ControlFlowNode for None | +| test.py:19 from import | test.py:3:5:3:5 | ControlFlowNode for a | 22 | test.py:19:5:19:6 | ControlFlowNode for IntegerLiteral | +| test.py:19 from import | test.py:4:5:4:5 | ControlFlowNode for b | None | test.py:19:9:19:12 | ControlFlowNode for None | +| test.py:20 from import | test.py:3:5:3:5 | ControlFlowNode for a | 23 | test.py:20:5:20:6 | ControlFlowNode for IntegerLiteral | +| test.py:20 from import | test.py:4:5:4:5 | ControlFlowNode for b | True | test.py:20:9:20:12 | ControlFlowNode for True | +| test.py:21 from import | test.py:3:5:3:5 | ControlFlowNode for a | 24 | test.py:21:5:21:6 | ControlFlowNode for IntegerLiteral | +| test.py:21 from import | test.py:4:5:4:5 | ControlFlowNode for b | '' | test.py:21:9:21:10 | ControlFlowNode for Str | +| test.py:22 from import | test.py:3:5:3:5 | ControlFlowNode for a | 3 | test.py:22:12:22:12 | ControlFlowNode for IntegerLiteral | +| test.py:22 from import | test.py:4:5:4:5 | ControlFlowNode for b | 7 | test.py:22:7:22:7 | ControlFlowNode for IntegerLiteral | diff --git a/python/ql/test/library-tests/PointsTo/global/Global.ql b/python/ql/test/library-tests/PointsTo/global/Global.ql new file mode 100644 index 000000000000..77971b12b0c8 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/global/Global.ql @@ -0,0 +1,14 @@ + +import python + + +import python +import semmle.python.pointsto.PointsTo2 +import semmle.python.pointsto.PointsToContext2 +import semmle.python.objects.ObjectInternal + +from ControlFlowNode f, PointsToContext2 ctx, ObjectInternal obj, ControlFlowNode orig +where exists(ExprStmt s | s.getValue().getAFlowNode() = f) and +PointsTo2::points_to(f, ctx, obj, orig) + +select ctx, f, obj.toString(), orig diff --git a/python/ql/test/library-tests/PointsTo/global/test.py b/python/ql/test/library-tests/PointsTo/global/test.py new file mode 100644 index 000000000000..07249d1cbd7f --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/global/test.py @@ -0,0 +1,22 @@ + +def bar(a, b=None): + a + b + if b: + return b + else: + return a + +def foo(): + bar + bar(11) + bar(12, None) + bar(13, True) + bar(14, "") + +bar +bar(21) +bar(22, None) +bar(23, True) +bar(24, "") +bar(b=7, a=3) diff --git a/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.expected b/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.expected index 6de862cee891..d4acab646d3a 100644 --- a/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.expected +++ b/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.expected @@ -11,3 +11,12 @@ | test.py:33:5:33:9 | ControlFlowNode for UnaryExpr | True | test.py:33:5:33:9 | ControlFlowNode for UnaryExpr | | test.py:38:9:38:9 | ControlFlowNode for f | False | test.py:20:9:20:13 | ControlFlowNode for False | | test.py:41:9:41:9 | ControlFlowNode for t | True | test.py:19:9:19:12 | ControlFlowNode for True | +| test.py:49:5:49:6 | ControlFlowNode for IntegerLiteral | 17 | test.py:49:5:49:6 | ControlFlowNode for IntegerLiteral | +| test.py:50:5:50:6 | ControlFlowNode for UnaryExpr | -3 | test.py:50:5:50:6 | ControlFlowNode for UnaryExpr | +| test.py:52:5:52:9 | ControlFlowNode for Compare | True | test.py:52:5:52:9 | ControlFlowNode for Compare | +| test.py:57:9:57:9 | ControlFlowNode for w | 2 | test.py:45:9:45:9 | ControlFlowNode for IntegerLiteral | +| test.py:58:9:58:9 | ControlFlowNode for z | 5 | test.py:48:9:48:9 | ControlFlowNode for IntegerLiteral | +| test.py:64:9:64:9 | ControlFlowNode for v | 20 | test.py:62:13:62:14 | ControlFlowNode for IntegerLiteral | +| test.py:66:9:66:9 | ControlFlowNode for v | 10 | test.py:60:13:60:14 | ControlFlowNode for IntegerLiteral | +| test.py:70:5:70:14 | ControlFlowNode for Str | 'a string' | test.py:70:5:70:14 | ControlFlowNode for Str | +| test.py:72:9:72:12 | ControlFlowNode for Str | ':)' | test.py:72:9:72:12 | ControlFlowNode for Str | diff --git a/python/ql/test/library-tests/PointsTo/local/test.py b/python/ql/test/library-tests/PointsTo/local/test.py index c830d2dd33da..5a657ce3adc6 100644 --- a/python/ql/test/library-tests/PointsTo/local/test.py +++ b/python/ql/test/library-tests/PointsTo/local/test.py @@ -1,4 +1,4 @@ -#Global + n = None t = True @@ -42,3 +42,34 @@ def foo(): else: f + w = 2 + x = 3 + y = 4 + z = 5 + 17 + -3 + 47164510561934056134523419250 # Too big + x < y + if x < w: + x + y + else: + w + z + if unknown(): + v = 10 + else: + v = 20 + if v > 15: + v + else: + v + +def strings(): + version = "3.8 alpha" + "a string" + if version >= "3": + ":)" + else: + ":(" + From bf692f4aade93d666796724d220ca6c7c2f7821f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 20 Mar 2019 14:35:41 +0000 Subject: [PATCH 004/128] Python: Add better class support, including inheritance. --- .../ql/src/semmle/python/objects/Classes.qll | 71 +++++-- .../src/semmle/python/objects/Constants.qll | 48 +++++ .../src/semmle/python/objects/Instances.qll | 100 +++++++++ .../ql/src/semmle/python/objects/Modules.qll | 40 +++- .../semmle/python/objects/ObjectInternal.qll | 102 +++++++-- .../ql/src/semmle/python/objects/TObject.qll | 40 +++- .../src/semmle/python/pointsto/PointsTo2.qll | 194 ++++++++++++++++++ .../library-tests/PointsTo/inheritance/Mro.ql | 11 +- .../objects/classes/Test.expected | 1 + .../library-tests/objects/classes/Test.ql | 8 + .../library-tests/objects/classes/test.py | 8 + 11 files changed, 569 insertions(+), 54 deletions(-) create mode 100644 python/ql/src/semmle/python/objects/Instances.qll create mode 100644 python/ql/test/library-tests/objects/classes/Test.expected create mode 100644 python/ql/test/library-tests/objects/classes/Test.ql create mode 100644 python/ql/test/library-tests/objects/classes/test.py diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index b82321d14106..4d2e2efdbff2 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -5,6 +5,7 @@ private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo2 private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.pointsto.MRO2 private import semmle.python.types.Builtins @@ -20,40 +21,37 @@ abstract class ClassObjectInternal extends ObjectInternal { override predicate notClass() { none() } - override predicate isComparable() { - any() + override int intValue() { + none() } - override predicate notComparable() { + override string strValue() { none() } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - // TO DO .. Result should (in most cases) be an instance - none() + string getName() { + result = this.getClassDeclaration().getName() } + abstract predicate attribute(string name, ObjectInternal value, CfgOrigin origin); + } class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject { Class getScope() { - exists(ClassDef def | - this = TPythonClassObject(def.getAFlowNode()) and - result = def.getDefinedClass() + exists(ClassExpr expr | + this = TPythonClassObject(expr.getAFlowNode()) and + result = expr.getInnerScope() ) } override string toString() { - result = this.getScope().toString() + result = "class " + this.getScope().getName() } override predicate introduced(ControlFlowNode node, PointsToContext2 context) { - exists(DefinitionNode def | - this = TPythonClassObject(def) and - node = def.getValue() and - context.appliesTo(node) - ) + this = TPythonClassObject(node) and context.appliesTo(node) } override ClassDecl getClassDeclaration() { @@ -61,7 +59,7 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject } override ObjectInternal getClass() { - result = TBuiltinClassObject(Builtin::special("FunctionType")) + result = Types::getMetaClass(this) } override Builtin getBuiltin() { @@ -80,6 +78,26 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject ) } + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + exists(ClassObjectInternal decl | + decl = Types::getMro(this).findDeclaringClass(name) | + Types::declaredAttribute(decl, name, value, origin) + ) + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // TO DO .. Result should (in most cases) be an instance + none() + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + } class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObject { @@ -108,4 +126,25 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec none() } + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + value.getBuiltin() = this.getBuiltin().getMember(name) and origin = CfgOrigin::unknown() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // TO DO .. Result should (in most cases) be an instance + none() + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + } diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index abb419203857..ee0253b6e019 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -48,6 +48,10 @@ abstract class BooleanObjectInternal extends ObjectInternal { none() } + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + } class TrueObjectInternal extends BooleanObjectInternal, TTrue { @@ -64,6 +68,14 @@ class TrueObjectInternal extends BooleanObjectInternal, TTrue { node.(NameNode).getId() = "True" and context.appliesTo(node) } + override int intValue() { + result = 1 + } + + override string strValue() { + none() + } + } class FalseObjectInternal extends BooleanObjectInternal, TFalse { @@ -80,6 +92,14 @@ class FalseObjectInternal extends BooleanObjectInternal, TFalse { node.(NameNode).getId() = "False" and context.appliesTo(node) } + override int intValue() { + result = 0 + } + + override string strValue() { + none() + } + } class NoneObjectInternal extends ObjectInternal, TNone { @@ -131,6 +151,18 @@ class NoneObjectInternal extends ObjectInternal, TNone { none() } + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + } @@ -184,12 +216,20 @@ class IntObjectInternal extends ObjectInternal, TInt { this = TInt(result) } + override string strValue() { + none() + } + override boolean booleanValue() { this.intValue() = 0 and result = false or this.intValue() != 0 and result = true } + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + } @@ -239,6 +279,10 @@ class StringObjectInternal extends ObjectInternal, TString { none() } + override int intValue() { + none() + } + override string strValue() { this = TString(result) } @@ -249,6 +293,10 @@ class StringObjectInternal extends ObjectInternal, TString { this.strValue() != "" and result = true } + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + } diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll new file mode 100644 index 000000000000..d310b2a9f441 --- /dev/null +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -0,0 +1,100 @@ +import python + + +private import semmle.python.objects.TObject +private import semmle.python.objects.ObjectInternal +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.types.Builtins + +class InstanceInternal extends TInstance, ObjectInternal { + + override string toString() { + result = "instance of " + this.getClass().(ClassObjectInternal).getClassDeclaration().getName() + } + + /** The boolean value of this object, if it has one */ + override boolean booleanValue() { + //this.getClass().instancesAlways(result) + none() + } + + /** Holds if this object may be true or false when evaluated as a bool */ + override predicate maybe() { + // this.getClass().instancesMaybe() + any() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + this = TInstance(node, _, context) + } + + /** Gets the class declaration for this object, if it is a declared class. */ + override ClassDecl getClassDeclaration() { + none() + } + + override predicate isClass() { + none() + } + + override predicate notClass() { + any() + } + + override ObjectInternal getClass() { + this = TInstance(_, result, _) + } + + /** Holds if whatever this "object" represents can be meaningfully analysed for + * truth or false in comparisons. For example, `None` or `int` can be, but `int()` + * or an unknown string cannot. + */ + override predicate isComparable() { + none() + } + + /** The negation of `isComparable()` */ + override predicate notComparable() { + any() + } + + /** Gets the `Builtin` for this object, if any. + * All objects (except unknown and undefined values) should return + * exactly one result for either this method or `getOrigin()`. + */ + override Builtin getBuiltin() { + none() + } + + /** Gets a control flow node that represents the source origin of this + * objects. + * All objects (except unknown and undefined values) should return + * exactly one result for either this method or `getBuiltin()`. + */ + override ControlFlowNode getOrigin() { + this = TInstance(result, _, _) + } + + /** Holds if `obj` is the result of calling `this` and `origin` is + * the origin of `obj`. + */ + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // In general instances aren't callable, but some are... + // TO DO -- Handle cases where class overrides __call__ + none() + } + + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + +} diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index d2d325a38858..923a0d3c8cb5 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -12,8 +12,6 @@ abstract class ModuleObjectInternal extends ObjectInternal { abstract Module getSourceModule(); - abstract predicate isBuiltin(); - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { // Modules aren't callable none() @@ -73,8 +71,16 @@ class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleOb none() } - override predicate isBuiltin() { - any() + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() } } @@ -142,10 +148,6 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { ) } - override predicate isBuiltin() { - none() - } - ModuleObjectInternal submodule(string name) { result.getName() = this.getName() + "." + name } @@ -157,6 +159,18 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { bl = 0 and bc = 0 and el = 0 and ec = 0 } + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + } class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { @@ -211,7 +225,15 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { result = TPythonModule(this.getSourceModule()) } - override predicate isBuiltin() { + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { none() } diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 8e9d13507df5..574d683d2c06 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -8,7 +8,7 @@ import semmle.python.objects.Modules import semmle.python.objects.Classes import semmle.python.objects.Constants -abstract class ObjectInternal extends TObject { +class ObjectInternal extends TObject { abstract string toString(); @@ -63,18 +63,18 @@ abstract class ObjectInternal extends TObject { /** The integer value of things that have integer values. * That is, ints and bools. */ - int intValue() { - none() - } + abstract int intValue(); /** The integer value of things that have integer values. * That is, strings. */ - string strValue() { - none() - } + abstract string strValue(); + + abstract predicate calleeAndOffset(Function scope, int paramOffset); - predicate calleeAndOffset(Function scope, int paramOffset) { none() } + final predicate isBuiltin() { + exists(this.getBuiltin()) + } } @@ -144,6 +144,14 @@ class PythonFunctionObjectInternal extends ObjectInternal, TPythonFunctionObject scope = this.getScope() and paramOffset = 0 } + override int intValue() { + none() + } + + override string strValue() { + none() + } + } /// BOUND METHODS @@ -200,6 +208,18 @@ class BuiltinFunctionObjectInternal extends ObjectInternal, TBuiltinFunctionObje none() } + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + } @@ -252,6 +272,18 @@ class BuiltinMethodObjectInternal extends ObjectInternal, TBuiltinMethodObject { none() } + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + } @@ -308,6 +340,18 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { none() } + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + } private predicate callee_for_object(PointsToContext2 callee, ObjectInternal obj) { @@ -318,18 +362,12 @@ private predicate callee_for_object(PointsToContext2 callee, ObjectInternal obj) } -class UnknownClassInternal extends ObjectInternal, TUnknownClass { +class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { override string toString() { none() } - override boolean booleanValue() { - none() - } - - override predicate maybe() { any() } - override ClassDecl getClassDeclaration() { result = Builtin::unknownType() } @@ -367,6 +405,14 @@ class UnknownClassInternal extends ObjectInternal, TUnknownClass { none() } + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + } class UnknownInternal extends ObjectInternal, TUnknown { @@ -418,6 +464,18 @@ class UnknownInternal extends ObjectInternal, TUnknown { none() } + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + } class UndefinedInternal extends ObjectInternal, TUndefined { @@ -471,6 +529,18 @@ class UndefinedInternal extends ObjectInternal, TUndefined { none() } + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + } module ObjectInternal { @@ -490,7 +560,7 @@ module ObjectInternal { result = TUnknown() } - ObjectInternal unknownClass() { + ClassObjectInternal unknownClass() { result = TUnknownClass() } diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index fe2d60cce76f..7d0218162c9a 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -1,5 +1,8 @@ import python private import semmle.python.types.Builtins +private import semmle.python.objects.ObjectInternal +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsToContext2 newtype TObject = TBuiltinClassObject(Builtin bltn) { @@ -22,8 +25,8 @@ newtype TObject = callable.getNode() instanceof CallableExpr } or - TPythonClassObject(ControlFlowNode classdef) { - classdef.getNode() instanceof ClassDef + TPythonClassObject(ControlFlowNode classexpr) { + classexpr.getNode() instanceof ClassExpr } or TPackageObject(Folder f) @@ -67,31 +70,52 @@ newtype TObject = s = quoted_string.regexpCapture("[bu]'([\\s\\S]*)'", 1) ) } - + or + TInstance(CallNode instantiation, ClassObjectInternal cls, PointsToContext2 context) { + PointsTo2::points_to(instantiation.getFunction(), context, cls, _) and + normal_class(cls) + } private predicate is_power_2(int n) { n = 1 or exists(int half | is_power_2(half) and n = half*2) } +private predicate normal_class(ClassObjectInternal cls) { + exists(Builtin bltn | + bltn = cls.getBuiltin() | + not bltn = Builtin::special(_) + ) + //or + //cls.getMro().inheritsFromType(false) +} + library class ClassDecl extends @py_object { ClassDecl() { this.(Builtin).isClass() and not this = Builtin::unknownType() or - this.(ControlFlowNode).getNode() instanceof ClassDef + this.(ControlFlowNode).getNode() instanceof ClassExpr } string toString() { result = "ClassDecl" } + private Class getClass() { + result = this.(ControlFlowNode).getNode().(ClassExpr).getInnerScope() + } + predicate declaresAttribute(string name) { exists(this.(Builtin).getMember(name)) or - exists(Class cls | - cls = this.(ControlFlowNode).getNode().(ClassDef).getDefinedClass() and - exists(SsaVariable var | name = var.getId() and var.getAUse() = cls.getANormalExit()) - ) + exists(SsaVariable var | name = var.getId() and var.getAUse() = this.getClass().getANormalExit()) + } + + string getName() { + result = this.(Builtin).getName() + or + result = this.getClass().getName() } + } diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index 7ae6b01e756f..6e9648a78f62 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -1,8 +1,11 @@ import python +private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.Filters private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.pointsto.MRO2 +private import semmle.python.types.Builtins /* Use this version for speed */ library class CfgOrigin extends @py_object { @@ -818,3 +821,194 @@ module Conditionals { } } + +module Types { + + int base_count(ClassObjectInternal cls) { + cls = ObjectInternal::builtin("object") and result = 0 + or + exists(cls.getBuiltin()) and cls != ObjectInternal::builtin("object") and result = 1 + or + exists(Class pycls | + pycls = cls.(PythonClassObjectInternal).getScope() | + result = strictcount(pycls.getABase()) + or + isNewStyle(cls) and not exists(pycls.getABase()) and result = 1 + or + isOldStyle(cls) and not exists(pycls.getABase()) and result = 0 + ) + } + + ClassObjectInternal getBase(ClassObjectInternal cls, int n) { + result.getBuiltin() = cls.getBuiltin().getBaseClass() and n = 0 + or + exists(Class pycls | + pycls = cls.(PythonClassObjectInternal).getScope() | + PointsTo2::points_to(pycls.getBase(n).getAFlowNode(), _, result, _) + or + not exists(pycls.getABase()) and n = 0 and + isNewStyle(cls) and result = ObjectInternal::builtin("object") + ) + or + cls = ObjectInternal::unknownClass() and n = 0 and + result = ObjectInternal::builtin("object") + } + + predicate isOldStyle(ClassObjectInternal cls) { + //To do... + none() + } + + predicate isNewStyle(ClassObjectInternal cls) { + //To do... + any() + } + + ClassList getMro(ClassObjectInternal cls) { + isNewStyle(cls) and + result = Mro::newStyleMro(cls) + or + // To do, old-style + none() + } + + predicate declaredAttribute(ClassObjectInternal cls, string name, ObjectInternal value, CfgOrigin origin) { + value.getBuiltin() = cls.getBuiltin().getMember(name) and origin = CfgOrigin::unknown() + or + value != ObjectInternal::undefined() and + exists(EssaVariable var | + name = var.getName() and + var.getAUse() = cls.(PythonClassObjectInternal).getScope().getANormalExit() and + PointsTo2::ssa_variable_points_to(var, _, value, origin) + ) + } + + ClassObjectInternal getMetaClass(PythonClassObjectInternal cls) { + result = declaredMetaClass(cls) + or + hasDeclaredMetaclass(cls) = false and result = getInheritedMetaclass(cls) + } + + private ClassObjectInternal declaredMetaClass(PythonClassObjectInternal cls) { + exists(ObjectInternal obj | + PointsTo2::ssa_variable_points_to(metaclass_var(cls.getScope()), _, obj, _) | + result = obj + or + obj = ObjectInternal::unknown() and result = ObjectInternal::unknownClass() + ) + or + exists(Builtin meta | + result.getBuiltin() = meta and + meta = cls.getBuiltin().getClass() and + meta.inheritsFromType() + ) + or + exists(ControlFlowNode meta | + six_add_metaclass(_, cls, meta) and + PointsTo2::points_to(meta, _, result, _) + ) + } + + private boolean hasDeclaredMetaclass(PythonClassObjectInternal cls) { + result = has_six_add_metaclass(cls).booleanOr(has_metaclass_var_metaclass(cls)) + } + + private boolean has_six_add_metaclass(PythonClassObjectInternal cls) { + // TO DO... + none() + } + + private boolean has_metaclass_var_metaclass(PythonClassObjectInternal cls) { + exists(ObjectInternal obj | + PointsTo2::ssa_variable_points_to(metaclass_var(cls.getScope()), _, obj, _) | + obj = ObjectInternal::undefined() and result = false + or + obj != ObjectInternal::undefined() and result = true + ) + or + exists(Class pycls | + pycls = cls.getScope() and + not exists(metaclass_var(pycls)) and result = false + ) + } + + private EssaVariable metaclass_var(Class cls) { + result.getASourceUse() = cls.getMetaClass().getAFlowNode() + or + major_version() = 2 and not exists(cls.getMetaClass()) and + result.getName() = "__metaclass__" and + cls.(ImportTimeScope).entryEdge(result.getAUse(), _) + } + + /** INTERNAL -- Do not use */ + cached predicate six_add_metaclass(CallNode decorator_call, ClassObjectInternal decorated, ControlFlowNode metaclass) { + //TO DO... + none() + //exists(CallNode decorator | + // decorator_call.getArg(0) = decorated and + // decorator = decorator_call.getFunction() and + // decorator.getArg(0) = metaclass | + // PointsTo2::points_to(decorator.getFunction(), _, six_add_metaclass_function(), _) + // or + // exists(ModuleObjectInternal six | + // six.getName() = "six" and + // PointsTo2::points_to(decorator.getFunction().(AttrNode).getObject("add_metaclass"), _, six, _) + // ) + //) + } + + private ObjectInternal six_add_metaclass_function() { + exists(Module six, FunctionExpr add_metaclass | + add_metaclass.getInnerScope().getName() = "add_metaclass" and + add_metaclass.getScope() = six and + result.getOrigin() = add_metaclass.getAFlowNode() + ) + } + + private ClassObjectInternal getInheritedMetaclass(ClassObjectInternal cls) { + result = getInheritedMetaclass(cls, 0) + or + // Best guess if base is not a known class + exists(ObjectInternal base | + base = getBase(cls, _) and + result = ObjectInternal::unknownClass() | + base.notClass() + or + base = ObjectInternal::unknownClass() + ) + } + + private ClassObjectInternal getInheritedMetaclass(ClassObjectInternal cls, int n) { + exists(Class c | + c = cls.(PythonClassObjectInternal).getScope() and + n = count(c.getABase()) + | + result = ObjectInternal::builtin("type") + ) + or + exists(ClassObjectInternal meta1, ClassObjectInternal meta2 | + meta1 = getMetaClass(getBase(cls, n)) and + meta2 = getInheritedMetaclass(cls, n+1) + | + /* Choose sub-class */ + improperSuperType(meta1) = meta2 and result = meta1 + or + improperSuperType(meta2) = meta1 and result = meta2 + or + /* Make sure we have a metaclass, even if base is unknown */ + meta1 = ObjectInternal::unknownClass() and result = ObjectInternal::builtin("type") + or + meta2 = ObjectInternal::unknownClass() and result = meta1 + ) + } + + private ClassObjectInternal improperSuperType(ClassObjectInternal cls) { + result = cls + or + result = improperSuperType(getBase(cls, _)) + } + +} + + + diff --git a/python/ql/test/library-tests/PointsTo/inheritance/Mro.ql b/python/ql/test/library-tests/PointsTo/inheritance/Mro.ql index d7373fc2b350..1b3543a23519 100644 --- a/python/ql/test/library-tests/PointsTo/inheritance/Mro.ql +++ b/python/ql/test/library-tests/PointsTo/inheritance/Mro.ql @@ -1,10 +1,11 @@ import python -/** Make unknown type visible */ -class UnknownType extends ClassObject { +private import semmle.python.objects.ObjectInternal +private import semmle.python.pointsto.PointsTo2 - UnknownType() { this = theUnknownType() } +/** Make unknown type visible */ +class UnknownType extends UnknownClassInternal { override string toString() { result = "*UNKNOWN TYPE" } @@ -12,6 +13,6 @@ class UnknownType extends ClassObject { } -from ClassObject c +from ClassObjectInternal c where not c.isBuiltin() -select c.toString(), c.getMro() +select c.toString(), Types::getMro(c) diff --git a/python/ql/test/library-tests/objects/classes/Test.expected b/python/ql/test/library-tests/objects/classes/Test.expected new file mode 100644 index 000000000000..9dae856c349e --- /dev/null +++ b/python/ql/test/library-tests/objects/classes/Test.expected @@ -0,0 +1 @@ +fail diff --git a/python/ql/test/library-tests/objects/classes/Test.ql b/python/ql/test/library-tests/objects/classes/Test.ql new file mode 100644 index 000000000000..bfea87939242 --- /dev/null +++ b/python/ql/test/library-tests/objects/classes/Test.ql @@ -0,0 +1,8 @@ +import python + +private import semmle.python.objects.ObjectInternal + +from ClassObjectInternal cls, ControlFlowNode f +where cls.introduced(f, _) +select cls.getName(), f + diff --git a/python/ql/test/library-tests/objects/classes/test.py b/python/ql/test/library-tests/objects/classes/test.py new file mode 100644 index 000000000000..b3ce272e850e --- /dev/null +++ b/python/ql/test/library-tests/objects/classes/test.py @@ -0,0 +1,8 @@ + + +class Foo(object): + pass + +class Bar(object): + pass + From 39b9723054a808ac58682ac08a4c883dd3a3539d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 20 Mar 2019 16:45:05 +0000 Subject: [PATCH 005/128] Python: Add support for bound-methods. --- .../src/semmle/python/objects/Callables.qll | 243 +++++++++ .../ql/src/semmle/python/objects/Classes.qll | 55 ++ .../src/semmle/python/objects/Instances.qll | 2 +- .../semmle/python/objects/ObjectInternal.qll | 270 +--------- .../ql/src/semmle/python/objects/TObject.qll | 39 +- python/ql/src/semmle/python/pointsto/MRO2.qll | 476 ++++++++++++++++++ .../python/pointsto/PointsToContext2.qll | 8 +- .../PointsTo/instances/LocalPointsTo.expected | 0 .../PointsTo/instances/LocalPointsTo.ql | 10 + .../library-tests/PointsTo/instances/test.py | 16 + .../PointsTo/methods/PointsTo.expected | 9 + .../PointsTo/methods/PointsTo.ql | 10 + .../library-tests/PointsTo/methods/test.py | 27 + .../library-tests/objects/classes/Test.ql | 1 + 14 files changed, 882 insertions(+), 284 deletions(-) create mode 100644 python/ql/src/semmle/python/objects/Callables.qll create mode 100644 python/ql/src/semmle/python/pointsto/MRO2.qll create mode 100644 python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.expected create mode 100644 python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.ql create mode 100644 python/ql/test/library-tests/PointsTo/instances/test.py create mode 100644 python/ql/test/library-tests/PointsTo/methods/PointsTo.expected create mode 100644 python/ql/test/library-tests/PointsTo/methods/PointsTo.ql create mode 100644 python/ql/test/library-tests/PointsTo/methods/test.py diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll new file mode 100644 index 000000000000..367fb818d69b --- /dev/null +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -0,0 +1,243 @@ + +import python + +private import semmle.python.objects.TObject +private import semmle.python.objects.ObjectInternal +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.pointsto.MRO2 +private import semmle.python.types.Builtins + + +abstract class CallableObjectInternal extends ObjectInternal { + + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate isClass() { none() } + + override predicate notClass() { any() } + + /** The boolean value of this object, if it has one */ + override boolean booleanValue() { + result = true + } + + /** Holds if this object may be true or false when evaluated as a bool */ + override predicate maybe() { none() } + + override ClassDecl getClassDeclaration() { + none() + } + + abstract string getName(); +} + + +class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFunctionObject { + + Function getScope() { + exists(CallableExpr expr | + this = TPythonFunctionObject(expr.getAFlowNode()) and + result = expr.getInnerScope() + ) + } + + override string toString() { + result = this.getScope().toString() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + this = TPythonFunctionObject(node) and context.appliesTo(node) + } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("FunctionType")) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override Builtin getBuiltin() { + none() + } + + override ControlFlowNode getOrigin() { + this = TPythonFunctionObject(result) + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + exists(Function func, ControlFlowNode rval | + func = this.getScope() and + callee.appliesToScope(func) and + rval = func.getAReturnValueFlowNode() and + PointsTo2::points_to(rval, callee, obj, origin) + ) + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + scope = this.getScope() and paramOffset = 0 + } + + override string getName() { + result = this.getScope().getName() + } + +} + +class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunctionObject { + + override Builtin getBuiltin() { + this = TBuiltinFunctionObject(result) + } + + override string toString() { + result = "builtin function " + this.getBuiltin().getName() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. + none() + } + + override ControlFlowNode getOrigin() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + + override string getName() { + result = this.getBuiltin().getName() + } + +} + + +class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethodObject { + + override Builtin getBuiltin() { + this = TBuiltinMethodObject(result) + } + + override string toString() { + result = "builtin method " + this.getBuiltin().getName() + } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(this.getBuiltin().getClass()) + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate isComparable() { + any() + } + + override predicate notComparable() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. + none() + } + + override ControlFlowNode getOrigin() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + + override string getName() { + result = this.getBuiltin().getName() + } + +} + +class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { + + override Builtin getBuiltin() { + none() + } + + CallableObjectInternal getFunction() { + this = TBoundMethod(_, _, result, _) + } + + ObjectInternal getSelf() { + this = TBoundMethod(_, result, _, _) + } + + override string toString() { + result = "bound method '" + this.getFunction().getName() + "' of " + this.getSelf().toString() + } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("MethodType")) + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + this = TBoundMethod(node, _, _, context) + } + + override predicate isComparable() { + none() + } + + override predicate notComparable() { + any() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + this.getFunction().callResult(callee, obj, origin) + } + + override ControlFlowNode getOrigin() { + this = TBoundMethod(result, _, _, _) + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + this.getFunction().calleeAndOffset(scope, paramOffset-1) + } + + override string getName() { + result = this.getFunction().getName() + } + +} + + + + diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 4d2e2efdbff2..17db029310c9 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -35,6 +35,9 @@ abstract class ClassObjectInternal extends ObjectInternal { abstract predicate attribute(string name, ObjectInternal value, CfgOrigin origin); + boolean isSpecial() { + result = Types::getMro(this).isSpecial() + } } class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject { @@ -148,3 +151,55 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec } } + + +class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { + + override string toString() { + none() + } + + override ClassDecl getClassDeclaration() { + result = Builtin::unknownType() + } + + override ObjectInternal getClass() { + result = TUnknownClass() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + override predicate isComparable() { + none() + } + + override predicate notComparable() { + any() + } + + override Builtin getBuiltin() { + none() + } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and + callee_for_object(callee, this) + } + + override ControlFlowNode getOrigin() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + +} + + diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index d310b2a9f441..6ccd8824f754 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -10,7 +10,7 @@ private import semmle.python.types.Builtins class InstanceInternal extends TInstance, ObjectInternal { override string toString() { - result = "instance of " + this.getClass().(ClassObjectInternal).getClassDeclaration().getName() + result = "instance of " + this.getClass().(ClassObjectInternal).getName() } /** The boolean value of this object, if it has one */ diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 574d683d2c06..bd4ffd414598 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -6,6 +6,8 @@ private import semmle.python.pointsto.PointsToContext2 private import semmle.python.types.Builtins import semmle.python.objects.Modules import semmle.python.objects.Classes +import semmle.python.objects.Instances +import semmle.python.objects.Callables import semmle.python.objects.Constants class ObjectInternal extends TObject { @@ -79,214 +81,6 @@ class ObjectInternal extends TObject { } -class PythonFunctionObjectInternal extends ObjectInternal, TPythonFunctionObject { - - Function getScope() { - exists(CallableExpr expr | - this = TPythonFunctionObject(expr.getAFlowNode()) and - result = expr.getInnerScope() - ) - } - - override string toString() { - result = this.getScope().toString() - } - - /** The boolean value of this object, if it has one */ - override boolean booleanValue() { - result = true - } - - /** Holds if this object may be true or false when evaluated as a bool */ - override predicate maybe() { none() } - - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { - this = TPythonFunctionObject(node) and context.appliesTo(node) - } - - /** INTERNAL */ - override ClassDecl getClassDeclaration() { none() } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(Builtin::special("FunctionType")) - } - - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - - override Builtin getBuiltin() { - none() - } - - override ControlFlowNode getOrigin() { - this = TPythonFunctionObject(result) - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - exists(Function func, ControlFlowNode rval | - func = this.getScope() and - callee.appliesToScope(func) and - rval = func.getAReturnValueFlowNode() and - PointsTo2::points_to(rval, callee, obj, origin) - ) - } - - override predicate calleeAndOffset(Function scope, int paramOffset) { - scope = this.getScope() and paramOffset = 0 - } - - override int intValue() { - none() - } - - override string strValue() { - none() - } - -} - -/// BOUND METHODS -/// -/// -/// - -class BuiltinFunctionObjectInternal extends ObjectInternal, TBuiltinFunctionObject { - - override Builtin getBuiltin() { - this = TBuiltinFunctionObject(result) - } - - override string toString() { - result = "builtin function " + this.getBuiltin().getName() - } - - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { - none() - } - - override predicate maybe() { none() } - - override ClassDecl getClassDeclaration() { - none() - } - - override boolean booleanValue() { - result = true - } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(this.getBuiltin().getClass()) - } - - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. - none() - } - - override ControlFlowNode getOrigin() { - none() - } - - override int intValue() { - none() - } - - override string strValue() { - none() - } - - override predicate calleeAndOffset(Function scope, int paramOffset) { - none() - } - -} - - -class BuiltinMethodObjectInternal extends ObjectInternal, TBuiltinMethodObject { - - override Builtin getBuiltin() { - this = TBuiltinMethodObject(result) - } - - override string toString() { - result = "builtin method " + this.getBuiltin().getName() - } - - override boolean booleanValue() { - result = true - } - - override predicate maybe() { none() } - - override ClassDecl getClassDeclaration() { - none() - } - - override predicate isClass() { none() } - - override predicate notClass() { any() } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(this.getBuiltin().getClass()) - } - - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { - none() - } - - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. - none() - } - - override ControlFlowNode getOrigin() { - none() - } - - override int intValue() { - none() - } - - override string strValue() { - none() - } - - override predicate calleeAndOffset(Function scope, int paramOffset) { - none() - } - -} - - class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { override Builtin getBuiltin() { @@ -354,66 +148,6 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { } -private predicate callee_for_object(PointsToContext2 callee, ObjectInternal obj) { - exists(CallNode call, PointsToContext2 caller | - callee.fromCall(call, caller) and - PointsTo2::points_to(call.getFunction(), caller, obj, _) - ) -} - - -class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { - - override string toString() { - none() - } - - override ClassDecl getClassDeclaration() { - result = Builtin::unknownType() - } - - override predicate isClass() { any() } - - override predicate notClass() { none() } - - override ObjectInternal getClass() { - result = TUnknownClass() - } - - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { - none() - } - - override predicate isComparable() { - none() - } - - override predicate notComparable() { - any() - } - - override Builtin getBuiltin() { - none() - } - - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and - callee_for_object(callee, this) - } - - override ControlFlowNode getOrigin() { - none() - } - - override predicate calleeAndOffset(Function scope, int paramOffset) { - none() - } - - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - none() - } - -} class UnknownInternal extends ObjectInternal, TUnknown { diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 7d0218162c9a..ea534ad68c93 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -73,7 +73,15 @@ newtype TObject = or TInstance(CallNode instantiation, ClassObjectInternal cls, PointsToContext2 context) { PointsTo2::points_to(instantiation.getFunction(), context, cls, _) and - normal_class(cls) + cls.isSpecial() = false + } + or + TBoundMethod(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext2 context) { + exists(ControlFlowNode objnode, string name | + objnode = instantiation.getObject(name) and + PointsTo2::points_to(objnode, context, self, _) and + self.getClass().(ClassObjectInternal).attribute(name, function, _) + ) } private predicate is_power_2(int n) { @@ -81,14 +89,6 @@ private predicate is_power_2(int n) { exists(int half | is_power_2(half) and n = half*2) } -private predicate normal_class(ClassObjectInternal cls) { - exists(Builtin bltn | - bltn = cls.getBuiltin() | - not bltn = Builtin::special(_) - ) - //or - //cls.getMro().inheritsFromType(false) -} library class ClassDecl extends @py_object { @@ -118,4 +118,25 @@ library class ClassDecl extends @py_object { result = this.getClass().getName() } + /** Whether this is a class whose instances we treat specially, rather than as a generic instance. + */ + predicate isSpecial() { + exists(string name | + this = Builtin::special(name) | + not name = "object" and + not name = "set" and + not name.matches("%Exception") and + not name.matches("%Error") + ) + } + +} + + +predicate callee_for_object(PointsToContext2 callee, ObjectInternal obj) { + exists(CallNode call, PointsToContext2 caller | + callee.fromCall(call, caller) and + PointsTo2::points_to(call.getFunction(), caller, obj, _) + ) } + diff --git a/python/ql/src/semmle/python/pointsto/MRO2.qll b/python/ql/src/semmle/python/pointsto/MRO2.qll new file mode 100644 index 000000000000..65b5a8ed0df5 --- /dev/null +++ b/python/ql/src/semmle/python/pointsto/MRO2.qll @@ -0,0 +1,476 @@ +/** Classes and predicates for computing the Method Resolution Order (MRO) of classes. + * Supports both old-style (diamond) inheritance and new-style (C3 linearization) inheritance. + */ + +/* + * Implementation of the C3 linearization algorithm. + * See https://en.wikipedia.org/wiki/C3_linearization + * + * The key operation is merge, which takes a list of lists and produces a list. + * We implement it as the method `ClassListList.merge()` + * + * To support that we need to determine the best candidate to extract from a list of lists, + * implemented as `ClassListList.bestMergeCandidate()` + * + * The following code is designed to implement those operations + * without negation and as efficiently as possible. + */ + +import python + +private import semmle.python.objects.TObject +private import semmle.python.objects.ObjectInternal +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.types.Builtins + + +cached private newtype TClassList = Empty() + or + Cons(ClassObjectInternal head, TClassList tail) { + required_cons(head, tail) + } + +/* Keep ClassList finite and as small as possible */ +private predicate required_cons(ClassObjectInternal head, ClassList tail) { + tail = merge_of_linearization_of_bases(head) + or + exists(ClassObjectInternal cls, int n | + head = Types::getBase(cls, n) and tail = bases(cls, n+1) + ) + or + head = ObjectInternal::builtin("object") and tail = Empty() + or + reverse_step(_, Cons(head, _), tail) + or + exists(ClassListList list | + merge_step(tail, list, _) and + head = list.bestMergeCandidate() + ) + or + exists(ClassList list, int n | + n = list.firstIndex(head) and + tail = list.deduplicate(n+1) + ) + or + exists(ClassListList list, int n | + head = list.getHead().getItem(n) and + tail = flatten_list(list, n+1) + ) + or + tail = list_old_style_base_mros(head).flatten() +} + +/** A list of classes, used to represent the MRO of a class */ +class ClassList extends TClassList { + + string toString() { + result = "[" + this.contents() + "]" + } + + string contents() { + this = Empty() and result = "" + or + exists(ClassObjectInternal head | + head = this.getHead() | + this.getTail() = Empty() and result = head.getName() + or + this.getTail() != Empty() and result = head.getName() + ", " + this.getTail().contents() + ) + } + + int length() { + this = Empty() and result = 0 + or + result = this.getTail().length() + 1 + } + + ClassObjectInternal getHead() { + this = Cons(result, _) + } + + ClassList getTail() { + this = Cons(_, result) + } + + ClassObjectInternal getItem(int n) { + n = 0 and result = this.getHead() + or + result = this.getTail().getItem(n-1) + } + + pragma [inline] + ClassList removeHead(ClassObjectInternal cls) { + this.getHead() = cls and result = this.getTail() + or + this.getHead() != cls and result = this + or + this = Empty() and result = Empty() + } + + predicate legalMergeHead(ClassObjectInternal cls) { + this.getTail().doesNotContain(cls) + or + this = Empty() + } + + /** Use negative formulation for efficiency */ + predicate contains(ClassObjectInternal cls) { + cls = this.getHead() + or + this.getTail().contains(cls) + } + + /** Use negative formulation to avoid negative recursion */ + predicate doesNotContain(ClassObjectInternal cls) { + this.relevantForContains(cls) and + cls != this.getHead() and + this.getTail().doesNotContain(cls) + or + this = Empty() + } + + private predicate relevantForContains(ClassObjectInternal cls) { + exists(ClassListList list | + list.getItem(_).getHead() = cls and + list.getItem(_) = this + ) + or + exists(ClassList l | + l.relevantForContains(cls) and + this = l.getTail() + ) + } + + ClassObjectInternal findDeclaringClass(string name) { + exists(ClassDecl head | + head = this.getHead().getClassDeclaration() | + if head.declaresAttribute(name) then + result = this.getHead() + else + result = this.getTail().findDeclaringClass(name) + ) + } + + predicate declares(string name) { + this.getHead().getClassDeclaration().declaresAttribute(name) + or + this.getTail().declares(name) + } + + ClassList startingAt(ClassObjectInternal cls) { + exists(ClassObjectInternal head | + head = this.getHead() | + if head = cls then + result = this + else + result = this.getTail().startingAt(cls) + ) + } + + ClassList deduplicate() { + result = this.deduplicate(0) + } + + /* Helpers for `deduplicate()` */ + + int firstIndex(ClassObjectInternal cls) { + result = this.firstIndex(cls, 0) + } + + /* Helper for firstIndex(cls), getting the first index of `cls` where result >= n */ + private int firstIndex(ClassObjectInternal cls, int n) { + this.getItem(n) = cls and result = n + or + this.getItem(n) != cls and result = this.firstIndex(cls, n+1) + } + + /** Holds if the class at `n` is a duplicate of an earlier position. */ + private predicate duplicate(int n) { + exists(ClassObjectInternal cls | + cls = this.getItem(n) and this.firstIndex(cls) < n + ) + } + + /** Gets a class list which is the de-duplicated form of the list containing elements of + * this list from `n` onwards. + */ + ClassList deduplicate(int n) { + n = this.length() and result = Empty() + or + this.duplicate(n) and result = this.deduplicate(n+1) + or + exists(ClassObjectInternal cls | + n = this.firstIndex(cls) and + result = Cons(cls, this.deduplicate(n+1)) + ) + } + + predicate isEmpty() { + this = Empty() + } + + ClassList reverse() { + reverse_step(this, Empty(), result) + } + + boolean isSpecial() { + this = Empty() and result = false + or + exists(ClassDecl decl | + decl = this.getHead().getClassDeclaration() | + if decl.isSpecial() then + result = true + else + result = this.getTail().isSpecial() + ) + } + +} + +private newtype TClassListList = + EmptyList() or + ConsList(TClassList head, TClassListList tail) { + required_list(head, tail) + } + +/* Keep ClassListList finite and as small as possible */ +private predicate required_list(ClassList head, ClassListList tail) { + any(ClassListList x).removedClassParts(_, head, tail, _) + or + head = bases(_) and tail = EmptyList() + or + exists(ClassObjectInternal cls, int n | + head = Mro::newStyleMro(Types::getBase(cls, n)) and + tail = list_of_linearization_of_bases_plus_bases(cls, n+1) + ) + or + exists(ClassObjectInternal cls, int n | + head = Mro::oldStyleMro(Types::getBase(cls, n)) and + tail = list_old_style_base_mros(cls, n+1) + ) +} + +private class ClassListList extends TClassListList { + + string toString() { + result = "[" + this.contents() + "]" + } + + string contents() { + this = EmptyList() and result = "" + or + exists(ClassList head | + head = this.getHead() | + this.getTail() = EmptyList() and result = head.toString() + or + this.getTail() != EmptyList() and result = head.toString() + ", " + this.getTail().contents() + ) + } + + int length() { + this = EmptyList() and result = 0 + or + result = this.getTail().length() + 1 + } + + ClassList getHead() { + this = ConsList(result, _) + } + + ClassListList getTail() { + this = ConsList(_, result) + } + + ClassList getItem(int n) { + n = 0 and result = this.getHead() + or + result = this.getTail().getItem(n-1) + } + + private ClassObjectInternal getAHead() { + result = this.getHead().getHead() + or + result = this.getTail().getAHead() + } + + pragma [nomagic] + ClassList merge() { + exists(ClassList reversed | + merge_step(reversed, EmptyList(), this) and + result = reversed.reverse() + ) + or + this = EmptyList() and result = Empty() + } + + /* Join ordering helper */ + pragma [noinline] + predicate removedClassParts(ClassObjectInternal cls, ClassList removed_head, ClassListList removed_tail, int n) { + cls = this.bestMergeCandidate() and n = this.length()-1 and + removed_head = this.getItem(n).removeHead(cls) and removed_tail = EmptyList() + or + exists(ClassList prev_head, ClassListList prev_tail | + this.removedClassParts(cls, prev_head, prev_tail, n+1) and + removed_head = this.getItem(n).removeHead(cls) and + removed_tail = ConsList(prev_head, prev_tail) + ) + } + + ClassListList remove(ClassObjectInternal cls) { + exists(ClassList removed_head, ClassListList removed_tail | + this.removedClassParts(cls, removed_head, removed_tail, 0) and + result = ConsList(removed_head, removed_tail) + ) + or + this = EmptyList() and result = EmptyList() + } + + predicate legalMergeCandidate(ClassObjectInternal cls, int n) { + cls = this.getAHead() and n = this.length() + or + this.getItem(n).legalMergeHead(cls) and + this.legalMergeCandidate(cls, n+1) + } + + predicate legalMergeCandidate(ClassObjectInternal cls) { + this.legalMergeCandidate(cls, 0) + } + + predicate illegalMergeCandidate(ClassObjectInternal cls) { + cls = this.getAHead() and + this.getItem(_).getTail().contains(cls) + } + + ClassObjectInternal bestMergeCandidate(int n) { + exists(ClassObjectInternal head | + head = this.getItem(n).getHead() + | + legalMergeCandidate(head) and result = head + or + illegalMergeCandidate(head) and result = this.bestMergeCandidate(n+1) + ) + } + + ClassObjectInternal bestMergeCandidate() { + result = this.bestMergeCandidate(0) + } + + /** Gets a ClassList representing the this list of list flattened into a single list. + * Used for old-style MRO computation. + */ + ClassList flatten() { + this = EmptyList() and result = Empty() + or + result = flatten_list(this, 0) + } + +} + +private ClassList flatten_list(ClassListList list, int n) { + need_flattening(list) and + exists(ClassList head, ClassListList tail | + list = ConsList(head, tail) + | + n = head.length() and result = tail.flatten() + or + result = Cons(head.getItem(n), flatten_list(list, n+1)) + ) +} + +/* Restrict flattening to those lists that need to be flattened */ +private predicate need_flattening(ClassListList list) { + list = list_old_style_base_mros(_) + or + exists(ClassListList toflatten | + need_flattening(toflatten) and + list = toflatten.getTail() + ) +} + +private ClassList bases(ClassObjectInternal cls) { + result = bases(cls, 0) +} + +private ClassList bases(ClassObjectInternal cls, int n) { + result = Cons(Types::getBase(cls, n), bases(cls, n+1)) + or + result = Empty() and n = Types::base_count(cls) +} + +private ClassListList list_of_linearization_of_bases_plus_bases(ClassObjectInternal cls) { + result = list_of_linearization_of_bases_plus_bases(cls, 0) +} + +private ClassListList list_of_linearization_of_bases_plus_bases(ClassObjectInternal cls, int n) { + result = ConsList(bases(cls), EmptyList()) and n = Types::base_count(cls) + or + exists(ClassListList partial | + partial = list_of_linearization_of_bases_plus_bases(cls, n+1) and + result = ConsList(Mro::newStyleMro(Types::getBase(cls, n)), partial) + ) +} + +private ClassList merge_of_linearization_of_bases(ClassObjectInternal cls) { + result = list_of_linearization_of_bases_plus_bases(cls).merge() +} + +private ClassListList list_old_style_base_mros(ClassObjectInternal cls) { + result = list_old_style_base_mros(cls, 0) +} + +pragma [nomagic] +private ClassListList list_old_style_base_mros(ClassObjectInternal cls, int n) { + n = Types::base_count(cls) and result = EmptyList() + or + result = ConsList(Mro::oldStyleMro(Types::getBase(cls, n)), list_old_style_base_mros(cls, n+1)) +} + +/** Holds if the pair `reversed_mro`, `remaining_list` represents a step in the C3 merge operation + * of computing the C3 linearization of `original`. + */ +private predicate merge_step(ClassList reversed_mro, ClassListList remaining_list, ClassListList original) { + remaining_list = list_of_linearization_of_bases_plus_bases(_) and reversed_mro = Empty() and remaining_list = original + or + /* Removes the best merge candidate from `remaining_list` and prepends it to `reversed_mro` */ + exists(ClassObjectInternal head, ClassList prev_reverse_mro, ClassListList prev_list | + merge_step(prev_reverse_mro, prev_list, original) and + head = prev_list.bestMergeCandidate() and + reversed_mro = Cons(head, prev_reverse_mro) and + remaining_list = prev_list.remove(head) + ) + or + merge_step(reversed_mro, ConsList(Empty(), remaining_list), original) +} + +/* Helpers for `ClassList.reverse()` */ + +private predicate needs_reversing(ClassList lst) { + merge_step(lst, EmptyList(), _) + or + lst = Empty() +} + +private predicate reverse_step(ClassList lst, ClassList remainder, ClassList reversed) { + needs_reversing(lst) and remainder = lst and reversed = Empty() + or + exists(ClassObjectInternal head, ClassList tail | + reversed = Cons(head, tail) and + reverse_step(lst, Cons(head, remainder), tail) + ) +} + +module Mro { + + cached ClassList newStyleMro(ClassObjectInternal cls) { + cls = ObjectInternal::builtin("object") and result = Cons(cls, Empty()) + or + result = Cons(cls, merge_of_linearization_of_bases(cls)) + } + + cached ClassList oldStyleMro(ClassObjectInternal cls) { + Types::isOldStyle(cls) and + result = Cons(cls, list_old_style_base_mros(cls).flatten()).(ClassList).deduplicate() + } + +} \ No newline at end of file diff --git a/python/ql/src/semmle/python/pointsto/PointsToContext2.qll b/python/ql/src/semmle/python/pointsto/PointsToContext2.qll index 2e3cd1d80a7b..4473ab8f30f0 100644 --- a/python/ql/src/semmle/python/pointsto/PointsToContext2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsToContext2.qll @@ -174,12 +174,8 @@ class PointsToContext2 extends TPointsToContext { this = TCallContext(call, outerContext, _) and s = func.getScope() ) - // TO DO... - //or - //exists(FunctionObject func | - // PointsTo2::Flow::callsite_calls_function(_, _, func, this, _) and - // s = func.getFunction() - //) + or + InterProceduralPointsTo::callsite_calls_function(_, _, s, this, _) } /** Holds if this context can apply to the CFG node `n`. */ diff --git a/python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.expected b/python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.expected new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.ql b/python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.ql new file mode 100644 index 000000000000..8b14766b5c48 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.ql @@ -0,0 +1,10 @@ + +import python +import semmle.python.pointsto.PointsTo2 +import semmle.python.objects.ObjectInternal + +from ControlFlowNode f, ObjectInternal obj, ControlFlowNode orig +where // exists(ExprStmt s | s.getValue().getAFlowNode() = f) and +PointsTo2::points_to(f, _, obj, orig) + +select f, obj.toString(), orig diff --git a/python/ql/test/library-tests/PointsTo/instances/test.py b/python/ql/test/library-tests/PointsTo/instances/test.py new file mode 100644 index 000000000000..3b8117b6f6f8 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/instances/test.py @@ -0,0 +1,16 @@ + + +class Foo(object): + pass + +class Bar(object): + pass + +Foo +Bar + +f = Foo() +b = Bar() + +f +b diff --git a/python/ql/test/library-tests/PointsTo/methods/PointsTo.expected b/python/ql/test/library-tests/PointsTo/methods/PointsTo.expected new file mode 100644 index 000000000000..0a6012419a07 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/methods/PointsTo.expected @@ -0,0 +1,9 @@ +| test.py:10 ... fail +| test.py:11:5:11:7 | ControlFlowNode for Attribute | bound method 'm' of instance of C | test.py:11:5:11:7 | ControlFlowNode for Attribute | +| test.py:12:5:12:10 | ControlFlowNode for Attribute() | 3 | test.py:12:9:12:9 | ControlFlowNode for IntegerLiteral | +| test.py:16:5:16:5 | ControlFlowNode for t | bound method 'm' of instance of C | test.py:15:9:15:11 | ControlFlowNode for Attribute | +| test.py:17:5:17:8 | ControlFlowNode for t() | 4 | test.py:17:7:17:7 | ControlFlowNode for IntegerLiteral | +| test.py:24:1:24:1 | ControlFlowNode for D | class D | test.py:19:1:19:16 | ControlFlowNode for ClassExpr | +| test.py:25 ... fail +| test.py:26 ... fail +| test.py:27 ... fail diff --git a/python/ql/test/library-tests/PointsTo/methods/PointsTo.ql b/python/ql/test/library-tests/PointsTo/methods/PointsTo.ql new file mode 100644 index 000000000000..767736718626 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/methods/PointsTo.ql @@ -0,0 +1,10 @@ + +import python +import semmle.python.pointsto.PointsTo2 +import semmle.python.objects.ObjectInternal + +from ControlFlowNode f, ObjectInternal obj, ControlFlowNode orig +where exists(ExprStmt s | s.getValue().getAFlowNode() = f) and +PointsTo2::points_to(f, _, obj, orig) + +select f, obj.toString(), orig diff --git a/python/ql/test/library-tests/PointsTo/methods/test.py b/python/ql/test/library-tests/PointsTo/methods/test.py new file mode 100644 index 000000000000..858545eb7856 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/methods/test.py @@ -0,0 +1,27 @@ + +class C(object): + + def m(self, arg1): + return arg1 + +c = C() + +def func_and_method(): + C.m + c.m + c.m(3) + +def flow_bound_method(): + t = c.m + t + t(4) + +class D(object): + + @staticmethod + def foo(arg): + return arg +D +D.foo +D.foo(1) +D().foo(2) diff --git a/python/ql/test/library-tests/objects/classes/Test.ql b/python/ql/test/library-tests/objects/classes/Test.ql index bfea87939242..eeb464d17f07 100644 --- a/python/ql/test/library-tests/objects/classes/Test.ql +++ b/python/ql/test/library-tests/objects/classes/Test.ql @@ -1,6 +1,7 @@ import python private import semmle.python.objects.ObjectInternal +private import semmle.python.pointsto.PointsTo2 from ClassObjectInternal cls, ControlFlowNode f where cls.introduced(f, _) From ec151e9b02c4d39cfd95bcc759dc77205871b4f8 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 20 Mar 2019 17:36:27 +0000 Subject: [PATCH 006/128] Python points-to: Convert two pairs of predicates to methods on booleans. --- .../src/semmle/python/objects/Callables.qll | 36 ++------------ .../ql/src/semmle/python/objects/Classes.qll | 28 ++--------- .../src/semmle/python/objects/Constants.qll | 49 ++++--------------- .../src/semmle/python/objects/Instances.qll | 21 +------- .../ql/src/semmle/python/objects/Modules.qll | 40 ++------------- .../semmle/python/objects/ObjectInternal.qll | 48 ++++-------------- .../src/semmle/python/pointsto/PointsTo2.qll | 12 ++--- 7 files changed, 41 insertions(+), 193 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 367fb818d69b..45eeadda4f1a 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -19,9 +19,7 @@ abstract class CallableObjectInternal extends ObjectInternal { none() } - override predicate isClass() { none() } - - override predicate notClass() { any() } + override boolean isClass() { result = false } /** The boolean value of this object, if it has one */ override boolean booleanValue() { @@ -60,13 +58,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti result = TBuiltinClassObject(Builtin::special("FunctionType")) } - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } + override boolean isComparable() { result = true } override Builtin getBuiltin() { none() @@ -113,13 +105,7 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc result = TBuiltinClassObject(this.getBuiltin().getClass()) } - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } + override boolean isComparable() { result = true } override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. @@ -159,13 +145,7 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod none() } - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } + override boolean isComparable() { result = true } override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. @@ -212,13 +192,7 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { this = TBoundMethod(node, _, _, context) } - override predicate isComparable() { - none() - } - - override predicate notComparable() { - any() - } + override boolean isComparable() { result = false } override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { this.getFunction().callResult(callee, obj, origin) diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 17db029310c9..341268935be8 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -17,9 +17,7 @@ abstract class ClassObjectInternal extends ObjectInternal { override predicate maybe() { none() } - override predicate isClass() { any() } - - override predicate notClass() { none() } + override boolean isClass() { result = true } override int intValue() { none() @@ -93,13 +91,7 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject none() } - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } + override boolean isComparable() { result = true } } @@ -142,13 +134,7 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec none() } - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } + override boolean isComparable() { result = true } } @@ -171,13 +157,7 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { none() } - override predicate isComparable() { - none() - } - - override predicate notComparable() { - any() - } + override boolean isComparable() { result = false } override Builtin getBuiltin() { none() diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index ee0253b6e019..8e1ca309d599 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -19,22 +19,15 @@ abstract class BooleanObjectInternal extends ObjectInternal { none() } - override predicate isClass() { none() } + override boolean isClass() { result = false } + + override boolean isComparable() { result = true } - override predicate notClass() { any() } override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("bool")) } - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - override Builtin getBuiltin() { none() } @@ -118,9 +111,10 @@ class NoneObjectInternal extends ObjectInternal, TNone { none() } - override predicate isClass() { none() } + override boolean isClass() { result = false } + + override boolean isComparable() { result = true } - override predicate notClass() { any() } override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("NoneType")) @@ -130,14 +124,6 @@ class NoneObjectInternal extends ObjectInternal, TNone { node.(NameNode).getId() = "None" and context.appliesTo(node) } - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - override Builtin getBuiltin() { none() } @@ -183,21 +169,14 @@ class IntObjectInternal extends ObjectInternal, TInt { none() } - override predicate isClass() { none() } + override boolean isClass() { result = false } - override predicate notClass() { any() } + override boolean isComparable() { result = true } override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("int")) } - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } override Builtin getBuiltin() { none() @@ -250,22 +229,14 @@ class StringObjectInternal extends ObjectInternal, TString { none() } - override predicate isClass() { none() } + override boolean isClass() { result = false } - override predicate notClass() { any() } + override boolean isComparable() { result = true } override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("str")) } - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - override Builtin getBuiltin() { none() } diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 6ccd8824f754..1aaa001cff6d 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -34,31 +34,14 @@ class InstanceInternal extends TInstance, ObjectInternal { none() } - override predicate isClass() { - none() - } + override boolean isClass() { result = false } - override predicate notClass() { - any() - } + override boolean isComparable() { result = false } override ObjectInternal getClass() { this = TInstance(_, result, _) } - /** Holds if whatever this "object" represents can be meaningfully analysed for - * truth or false in comparisons. For example, `None` or `int` can be, but `int()` - * or an unknown string cannot. - */ - override predicate isComparable() { - none() - } - - /** The negation of `isComparable()` */ - override predicate notComparable() { - any() - } - /** Gets the `Builtin` for this object, if any. * All objects (except unknown and undefined values) should return * exactly one result for either this method or `getOrigin()`. diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 923a0d3c8cb5..2d533b5ca04b 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -21,6 +21,10 @@ abstract class ModuleObjectInternal extends ObjectInternal { result = this.getSourceModule().getEntryNode() } + override boolean isClass() { result = false } + + override boolean isComparable() { result = true } + } class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleObject { @@ -51,22 +55,10 @@ class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleOb none() } - override predicate isClass() { none() } - - override predicate notClass() { any() } - override ObjectInternal getClass() { result = TBuiltinClassObject(this.getBuiltin().getClass()) } - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - override Module getSourceModule() { none() } @@ -117,22 +109,10 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { none() } - override predicate isClass() { none() } - - override predicate notClass() { any() } - override ObjectInternal getClass() { result = TBuiltinClassObject(this.getBuiltin().getClass()) } - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - override Module getSourceModule() { result.getFile() = this.getFolder().getFile("__init__.py") } @@ -201,22 +181,10 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { none() } - override predicate isClass() { none() } - - override predicate notClass() { any() } - override ObjectInternal getClass() { result = TBuiltinClassObject(this.getBuiltin().getClass()) } - override predicate isComparable() { - any() - } - - override predicate notComparable() { - none() - } - override Module getSourceModule() { this = TPythonModule(result) } diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index bd4ffd414598..0c40995ef302 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -25,20 +25,16 @@ class ObjectInternal extends TObject { /** Gets the class declaration for this object, if it is a declared class. */ abstract ClassDecl getClassDeclaration(); - abstract predicate isClass(); - - abstract predicate notClass(); + /** True if this "object" is a class. */ + abstract boolean isClass(); abstract ObjectInternal getClass(); - /** Holds if whatever this "object" represents can be meaningfully analysed for + /** True if this "object" can be meaningfully analysed for * truth or false in comparisons. For example, `None` or `int` can be, but `int()` * or an unknown string cannot. */ - abstract predicate isComparable(); - - /** The negation of `isComparable()` */ - abstract predicate notComparable(); + abstract boolean isComparable(); /** Gets the `Builtin` for this object, if any. * All objects (except unknown and undefined values) should return @@ -105,9 +101,7 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { none() } - override predicate isClass() { none() } - - override predicate notClass() { any() } + override boolean isClass() { result = false } override ObjectInternal getClass() { result = TBuiltinClassObject(this.getBuiltin().getClass()) @@ -117,13 +111,7 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { none() } - override predicate isComparable() { - none() - } - - override predicate notComparable() { - any() - } + override boolean isComparable() { result = false } override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and @@ -165,9 +153,7 @@ class UnknownInternal extends ObjectInternal, TUnknown { none() } - override predicate isClass() { none() } - - override predicate notClass() { any() } + override boolean isClass() { result = false } override ObjectInternal getClass() { result = TUnknownClass() @@ -177,13 +163,7 @@ class UnknownInternal extends ObjectInternal, TUnknown { none() } - override predicate isComparable() { - none() - } - - override predicate notComparable() { - any() - } + override boolean isComparable() { result = false } override Builtin getBuiltin() { none() @@ -228,9 +208,9 @@ class UndefinedInternal extends ObjectInternal, TUndefined { none() } - override predicate isClass() { none() } + override boolean isClass() { result = false } - override predicate notClass() { any() } + override boolean isComparable() { result = false } override ObjectInternal getClass() { none() @@ -240,14 +220,6 @@ class UndefinedInternal extends ObjectInternal, TUndefined { none() } - override predicate isComparable() { - none() - } - - override predicate notComparable() { - any() - } - override Builtin getBuiltin() { none() } diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index 6e9648a78f62..d031579e3cb6 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -390,14 +390,14 @@ module PointsTo2 { equality_test(cmp, a, is, b) and points_to(a, context, o1, _) and points_to(b, context, o2, _) | - (o1.isComparable() and o2.isComparable()) and + (o1.isComparable() = true and o2.isComparable() = true) and ( o1 = o2 and value = ObjectInternal::bool(is) or o1 != o2 and value = ObjectInternal::bool(is.booleanNot()) ) or - (o1.notComparable() or o2.notComparable()) and + (o1.isComparable() = false or o2.isComparable() = false) and value = ObjectInternal::bool(_) ) or @@ -777,16 +777,16 @@ module Conditionals { exists(ObjectInternal other | PointsTo2::points_to(use, context, val, origin) and PointsTo2::points_to(r, context, other, _) | - val.isComparable() and other.isComparable() and + val.isComparable() = true and other.isComparable() = true and ( other = val and result = sense or other != val and result = sense.booleanNot() ) or - val.notComparable() and result = maybe() + val.isComparable() = false and result = maybe() or - other.notComparable() and result = maybe() + other.isComparable() = false and result = maybe() ) ) } @@ -972,7 +972,7 @@ module Types { exists(ObjectInternal base | base = getBase(cls, _) and result = ObjectInternal::unknownClass() | - base.notClass() + base.isClass() = false or base = ObjectInternal::unknownClass() ) From ce9d0f1a061036d62d61f781143780431742b000 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 21 Mar 2019 09:43:57 +0000 Subject: [PATCH 007/128] Python points-to: Add support for some more ESSA definitions. --- .../ql/src/semmle/python/objects/TObject.qll | 1 + .../src/semmle/python/pointsto/PointsTo2.qll | 65 ++++++++++++++++--- 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index ea534ad68c93..425dc55262b0 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -69,6 +69,7 @@ newtype TObject = quoted_string = bltn.getName() and s = quoted_string.regexpCapture("[bu]'([\\s\\S]*)'", 1) ) + or s = "__main__" } or TInstance(CallNode instantiation, ClassObjectInternal cls, PointsToContext2 context) { diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index d031579e3cb6..8d706e1c032e 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -85,6 +85,12 @@ module CfgOrigin { result = unknownValue() } + CfgOrigin fromModule(ModuleObjectInternal mod) { + mod.isBuiltin() and result = unknownValue() + or + result = mod.getSourceModule().getEntryNode() + } + } module PointsTo2 { @@ -264,14 +270,14 @@ module PointsTo2 { //// TO DO... // or // self_parameter_points_to(def, context, value, origin) - // or - // delete_points_to(def, context, value, origin) - // or - // module_name_points_to(def, context, value, origin) + or + delete_points_to(def, context, value, origin) + or + module_name_points_to(def, context, value, origin) or scope_entry_points_to(def, context, value, origin) - // or - // implicit_submodule_points_to(def, context, value, origin) + or + InterModulePointsTo::implicit_submodule_points_to(def, context, value, origin) // or // iteration_definition_points_to(def, context, value, origin) /* @@ -322,6 +328,35 @@ module PointsTo2 { points_to(def.getValue(), context, value, origin) } + /** Points-to for deletion: `del name`. */ + pragma [noinline] + private predicate delete_points_to(DeletionDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + value = ObjectInternal::undefined() and origin = def.getDefiningNode() and context.appliesToScope(def.getScope()) + } + + /** Implicit "definition" of `__name__` at the start of a module. */ + pragma [noinline] + private predicate module_name_points_to(ScopeEntryDefinition def, PointsToContext2 context, StringObjectInternal value, ControlFlowNode origin) { + def.getVariable().getName() = "__name__" and + exists(Module m | + m = def.getScope() + | + value = module_dunder_name(m) and context.isImport() + or + value.strValue() = "__main__" and context.isMain() and context.appliesToScope(m) + ) and + origin = def.getDefiningNode() + } + + private StringObjectInternal module_dunder_name(Module m) { + exists(string name | + result.strValue() = name | + if m.isPackageInit() then + name = m.getPackage().getName() + else + name = m.getName() + ) + } /** Holds if the phi-function `phi` refers to `(value, origin)` given the context `context`. */ pragma [nomagic] @@ -566,9 +601,21 @@ module InterModulePointsTo { exists(ModuleObjectInternal mod | mod = package.submodule(name) and value = mod | - origin = CfgOrigin::fromCfgNode(mod.getSourceModule().getEntryNode()) - or - mod.isBuiltin() and origin = CfgOrigin::unknown() + origin = CfgOrigin::fromModule(mod) + ) + } + + /** Implicit "definition" of the names of submodules at the start of an `__init__.py` file. + * + * PointsTo isn't exactly how the interpreter works, but is the best approximation we can manage statically. + */ + pragma [noinline] + predicate implicit_submodule_points_to(ImplicitSubModuleDefinition def, PointsToContext2 context, ModuleObjectInternal value, ControlFlowNode origin) { + exists(PackageObjectInternal package | + package.getSourceModule() = def.getDefiningNode().getScope() | + value = package.submodule(def.getSourceVariable().getName()) and + origin = CfgOrigin::fromModule(value).fix(def.getDefiningNode()) and + context.isImport() ) } From 84c9866c5014c28243b65854ca6f1cb0ce988032 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 21 Mar 2019 11:19:32 +0000 Subject: [PATCH 008/128] Python points-to: Add generic instances and handle returns for builtin functions. Move attribute lookup handling to objects. --- .../src/semmle/python/objects/Callables.qll | 44 ++++++++- .../ql/src/semmle/python/objects/Classes.qll | 12 ++- .../src/semmle/python/objects/Constants.qll | 24 +++++ .../src/semmle/python/objects/Instances.qll | 96 ++++++++++++++++++- .../ql/src/semmle/python/objects/Modules.qll | 61 ++++++++++++ .../src/semmle/python/objects/ObjectAPI.qll | 25 +++++ .../semmle/python/objects/ObjectInternal.qll | 38 +++++++- .../ql/src/semmle/python/objects/TObject.qll | 16 ++-- .../src/semmle/python/pointsto/PointsTo2.qll | 92 ++++-------------- .../ql/src/semmle/python/types/Builtins.qll | 16 ++++ 10 files changed, 337 insertions(+), 87 deletions(-) create mode 100644 python/ql/src/semmle/python/objects/ObjectAPI.qll diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 45eeadda4f1a..1b95813cd9ca 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -34,6 +34,13 @@ abstract class CallableObjectInternal extends ObjectInternal { } abstract string getName(); + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + override predicate attributesUnknown() { none() } + } @@ -108,8 +115,17 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc override boolean isComparable() { result = true } override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. - none() + exists(Builtin func, ClassObjectInternal cls | + func = this.getBuiltin() and + func != Builtin::builtin("isinstance") and + func != Builtin::builtin("issubclass") and + func != Builtin::builtin("callable") + | + cls = ObjectInternal::fromBuiltin(this.getReturnType()) and + obj = TUnknownInstance(cls) + ) and + origin = CfgOrigin::unknown() and + callee_for_object(callee, this) } override ControlFlowNode getOrigin() { @@ -124,6 +140,30 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc result = this.getBuiltin().getName() } + Builtin getReturnType() { + exists(Builtin func | + func = this.getBuiltin() | + /* Enumerate the types of a few builtin functions, that the CPython analysis misses. + */ + func = Builtin::builtin("hex") and result = Builtin::special("str") + or + func = Builtin::builtin("oct") and result = Builtin::special("str") + or + func = Builtin::builtin("intern") and result = Builtin::special("str") + or + /* Fix a few minor inaccuracies in the CPython analysis */ + ext_rettype(func, result) and not ( + func = Builtin::builtin("__import__") and result = Builtin::special("NoneType") + or + func = Builtin::builtin("compile") and result = Builtin::special("NoneType") + or + func = Builtin::builtin("sum") + or + func = Builtin::builtin("filter") + ) + ) + } + } diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 341268935be8..1d0ec77301ed 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -31,11 +31,10 @@ abstract class ClassObjectInternal extends ObjectInternal { result = this.getClassDeclaration().getName() } - abstract predicate attribute(string name, ObjectInternal value, CfgOrigin origin); - boolean isSpecial() { result = Types::getMro(this).isSpecial() } + } class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject { @@ -86,6 +85,8 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject ) } + override predicate attributesUnknown() { none() } + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { // TO DO .. Result should (in most cases) be an instance none() @@ -126,9 +127,12 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec } override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - value.getBuiltin() = this.getBuiltin().getMember(name) and origin = CfgOrigin::unknown() + value = ObjectInternal::fromBuiltin(this.getBuiltin().getMember(name)) and + origin = CfgOrigin::unknown() } + override predicate attributesUnknown() { none() } + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { // TO DO .. Result should (in most cases) be an instance none() @@ -180,6 +184,8 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { none() } + override predicate attributesUnknown() { any() } + } diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index 8e1ca309d599..7a9315fc210e 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -45,6 +45,12 @@ abstract class BooleanObjectInternal extends ObjectInternal { none() } + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + override predicate attributesUnknown() { none() } + } class TrueObjectInternal extends BooleanObjectInternal, TTrue { @@ -149,6 +155,12 @@ class NoneObjectInternal extends ObjectInternal, TNone { none() } + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + override predicate attributesUnknown() { none() } + } @@ -209,6 +221,12 @@ class IntObjectInternal extends ObjectInternal, TInt { none() } + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + override predicate attributesUnknown() { none() } + } @@ -268,6 +286,12 @@ class StringObjectInternal extends ObjectInternal, TString { none() } + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + override predicate attributesUnknown() { none() } + } diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 1aaa001cff6d..4c2df644d5e1 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -7,7 +7,7 @@ private import semmle.python.pointsto.PointsTo2 private import semmle.python.pointsto.PointsToContext2 private import semmle.python.types.Builtins -class InstanceInternal extends TInstance, ObjectInternal { +class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { override string toString() { result = "instance of " + this.getClass().(ClassObjectInternal).getName() @@ -26,7 +26,7 @@ class InstanceInternal extends TInstance, ObjectInternal { } override predicate introduced(ControlFlowNode node, PointsToContext2 context) { - this = TInstance(node, _, context) + this = TSpecificInstance(node, _, context) } /** Gets the class declaration for this object, if it is a declared class. */ @@ -39,7 +39,7 @@ class InstanceInternal extends TInstance, ObjectInternal { override boolean isComparable() { result = false } override ObjectInternal getClass() { - this = TInstance(_, result, _) + this = TSpecificInstance(_, result, _) } /** Gets the `Builtin` for this object, if any. @@ -56,7 +56,7 @@ class InstanceInternal extends TInstance, ObjectInternal { * exactly one result for either this method or `getBuiltin()`. */ override ControlFlowNode getOrigin() { - this = TInstance(result, _, _) + this = TSpecificInstance(result, _, _) } /** Holds if `obj` is the result of calling `this` and `origin` is @@ -80,4 +80,92 @@ class InstanceInternal extends TInstance, ObjectInternal { none() } + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + override predicate attributesUnknown() { any() } + +} + +/** Represents a value that has a known class, but no other information */ +class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { + + override string toString() { + result = "instance of " + this.getClass().(ClassObjectInternal).getName() + } + + /** The boolean value of this object, if it has one */ + override boolean booleanValue() { + //this.getClass().instancesAlways(result) + none() + } + + /** Holds if this object may be true or false when evaluated as a bool */ + override predicate maybe() { + // this.getClass().instancesMaybe() + any() + } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + none() + } + + /** Gets the class declaration for this object, if it is a declared class. */ + override ClassDecl getClassDeclaration() { + none() + } + + override boolean isClass() { result = false } + + override boolean isComparable() { result = false } + + override ObjectInternal getClass() { + this = TUnknownInstance(result) + } + + /** Gets the `Builtin` for this object, if any. + * All objects (except unknown and undefined values) should return + * exactly one result for either this method or `getOrigin()`. + */ + override Builtin getBuiltin() { + none() + } + + /** Gets a control flow node that represents the source origin of this + * objects. + * All objects (except unknown and undefined values) should return + * exactly one result for either this method or `getBuiltin()`. + */ + override ControlFlowNode getOrigin() { + none() + } + + /** Holds if `obj` is the result of calling `this` and `origin` is + * the origin of `obj`. + */ + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + // In general instances aren't callable, but some are... + // TO DO -- Handle cases where class overrides __call__ + none() + } + + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + override predicate attributesUnknown() { any() } + } diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 2d533b5ca04b..0e6529103587 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -75,6 +75,13 @@ class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleOb none() } + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + value = ObjectInternal::fromBuiltin(this.getBuiltin().getMember(name)) and + origin = CfgOrigin::unknown() + } + + override predicate attributesUnknown() { none() } + } class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { @@ -151,6 +158,40 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { none() } + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + this.getInitModule().attribute(name, value, origin) + or + // TO DO, dollar variable... + //exists(Module init | + // init = this.getSourceModule() and + // not exists(EssaVariable var | var.getAUse() = init.getANormalExit() and var.getSourceVariable().getName() = name) and + // exists(EssaVariable var, Context context | + // isModuleStateVariable(var) and var.getAUse() = init.getANormalExit() and + // context.isImport() and + // SSA::ssa_variable_named_attribute_points_to(var, context, name, undefinedVariable(), _, origin) and + // value = this.submodule(name) + // ) + //) + //or + this.hasNoInitModule() and + exists(ModuleObjectInternal mod | + mod = this.submodule(name) and + value = mod | + origin = CfgOrigin::fromModule(mod) + ) + } + + override predicate attributesUnknown() { none() } + +} + +/** Get the ESSA pseudo-variable used to retain module state + * during module initialization. Module attributes are handled + * as attributes of this variable, allowing the SSA form to track + * mutations of the module during its creation. + */ +private predicate isModuleStateVariable(EssaVariable var) { + var.getName() = "$" and var.getScope() instanceof Module } class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { @@ -205,5 +246,25 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { none() } + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + exists(EssaVariable var, ControlFlowNode exit, PointsToContext2 imp | + exit = this.getSourceModule().getANormalExit() and var.getAUse() = exit and + var.getSourceVariable().getName() = name and + PointsTo2::ssa_variable_points_to(var, imp, value, origin) and + imp.isImport() and + value != ObjectInternal::undefined() + ) + // TO DO, dollar variable... + //or + //not exists(EssaVariable var | var.getAUse() = m.getANormalExit() and var.getSourceVariable().getName() = name) and + //exists(EssaVariable var, PointsToContext2 imp | + // var.getAUse() = m.getANormalExit() and isModuleStateVariable(var) | + // PointsTo2::ssa_variable_named_attribute_points_to(var, imp, name, obj, origin) and + // imp.isImport() and obj != ObjectInternal::undefined() + //) + } + + override predicate attributesUnknown() { none() } + } diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll new file mode 100644 index 000000000000..77af652dd9c6 --- /dev/null +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -0,0 +1,25 @@ +import python +private import TObject +private import ObjectInternal +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsToContext2 + +class Value extends TObject { + + string toString() { + result = this.(ObjectInternal).toString() + } + + ControlFlowNode getAReferent() { + PointsTo2::points_to(result, _, this, _) + } + + predicate pointsTo(ControlFlowNode referent, PointsToContext2 context, ControlFlowNode origin) { + PointsTo2::points_to(referent, context, this, origin) + } + + Value getClass() { + result = this.(ObjectInternal).getClass() + } + +} diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 0c40995ef302..b9e3153527ef 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -74,6 +74,10 @@ class ObjectInternal extends TObject { exists(this.getBuiltin()) } + abstract predicate attribute(string name, ObjectInternal value, CfgOrigin origin); + + abstract predicate attributesUnknown(); + } @@ -84,7 +88,7 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { } override string toString() { - none() + result = this.getBuiltin().toString() } override boolean booleanValue() { @@ -134,6 +138,13 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { none() } + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + value = ObjectInternal::fromBuiltin(this.getBuiltin().getMember(name)) and + origin = CfgOrigin::unknown() + } + + override predicate attributesUnknown() { none() } + } @@ -190,6 +201,12 @@ class UnknownInternal extends ObjectInternal, TUnknown { none() } + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + override predicate attributesUnknown() { any() } + } class UndefinedInternal extends ObjectInternal, TUndefined { @@ -247,6 +264,12 @@ class UndefinedInternal extends ObjectInternal, TUndefined { none() } + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + override predicate attributesUnknown() { none() } + } module ObjectInternal { @@ -290,5 +313,18 @@ module ObjectInternal { result = TInt(n) } + ObjectInternal fromBuiltin(Builtin b) { + result = TInt(b.intValue()) + or + result = TString(b.strValue()) + or + result = TBuiltinClassObject(b) + or + result = TBuiltinFunctionObject(b) + or + result = TBuiltinOpaqueObject(b) + or + result = TBuiltinModuleObject(b) + } } diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 425dc55262b0..29957f87e0c0 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -18,6 +18,8 @@ newtype TObject = TBuiltinOpaqueObject(Builtin bltn) { not bltn.isClass() and not bltn.isFunction() and not bltn.isMethod() and not bltn.isModule() and + not exists(bltn.intValue()) and + not exists(bltn.strValue()) and not py_special_objects(bltn, _) } or @@ -56,6 +58,8 @@ newtype TObject = exists(UnaryExpr neg | neg.getOp() instanceof USub and neg.getOperand() = num) and n = -num.getN().toInt() ) + or + n = any(Builtin b).intValue() } or TString(string s) { @@ -65,14 +69,12 @@ newtype TObject = ) or // Any string from the library put in the DB by the extractor. - exists(string quoted_string, Builtin bltn | - quoted_string = bltn.getName() and - s = quoted_string.regexpCapture("[bu]'([\\s\\S]*)'", 1) - ) - or s = "__main__" + s = any(Builtin b).strValue() + or + s = "__main__" } or - TInstance(CallNode instantiation, ClassObjectInternal cls, PointsToContext2 context) { + TSpecificInstance(CallNode instantiation, ClassObjectInternal cls, PointsToContext2 context) { PointsTo2::points_to(instantiation.getFunction(), context, cls, _) and cls.isSpecial() = false } @@ -84,6 +86,8 @@ newtype TObject = self.getClass().(ClassObjectInternal).attribute(name, function, _) ) } + or + TUnknownInstance(ClassObjectInternal cls) { cls != TUnknownClass() } private predicate is_power_2(int n) { n = 1 or diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index 8d706e1c032e..183dd9df917d 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -104,6 +104,8 @@ module PointsTo2 { predicate points_to_candidate(ControlFlowNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { use_points_to(f, context, value, origin) or + attribute_load_points_to(f, context, value, origin) + or subscript_points_to(f, context, value, origin) or binary_expr_points_to(f, context, value, origin) @@ -242,6 +244,23 @@ module PointsTo2 { result.getSourceVariable() instanceof GlobalVariable } + /** Holds if `f` is an attribute `x.attr` and points to `(value, cls, origin)`. */ + private predicate attribute_load_points_to(AttrNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + exists(ObjectInternal object, string name, CfgOrigin orig | + points_to(f.getObject(name), context, object, _) | + object.attribute(name, value, orig) and + origin = orig.fix(f) + or + object.attributesUnknown() and origin = f and value = ObjectInternal::unknown() + ) + // TO DO -- Support CustomPointsToAttribute + //or + //exists(CustomPointsToAttribute object, string name | + // points_to(f.getObject(name), context, object, _, _) and + // object.attributePointsTo(name, value, cls, origin) + //) + } + /** Holds if the ESSA definition `def` refers to `(value, origin)` given the context `context`. */ predicate ssa_definition_points_to(EssaDefinition def, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { ssa_phi_points_to(def, context, value, origin) @@ -514,7 +533,7 @@ module InterModulePointsTo { //) //or (mod.getSourceModule() != f.getEnclosingModule() or mod.isBuiltin()) and - module_attribute_points_to(mod, name, value, orig) + mod.attribute(name, value, origin) ) or exists(EssaVariable var, CfgOrigin orig | @@ -551,60 +570,6 @@ module InterModulePointsTo { ) } - /** Holds if `mod.name` points to `(value, origin)`, where `mod` is a module object. */ - predicate module_attribute_points_to(ModuleObjectInternal mod, string name, ObjectInternal value, CfgOrigin origin) { - py_module_attributes(mod.getSourceModule(), name, value, origin) - or - package_attribute_points_to(mod, name, value, origin) - or - value.getBuiltin() = mod.getBuiltin().getMember(name) and - origin = CfgOrigin::unknown() - } - - /** Holds if `m.name` points to `(value, origin)`, where `m` is a (source) module. */ - cached predicate py_module_attributes(Module m, string name, ObjectInternal obj, CfgOrigin origin) { - exists(EssaVariable var, ControlFlowNode exit, PointsToContext2 imp | - exit = m.getANormalExit() and var.getAUse() = exit and - var.getSourceVariable().getName() = name and - PointsTo2::ssa_variable_points_to(var, imp, obj, origin) and - imp.isImport() and - obj != ObjectInternal::undefined() - ) - // TO DO, dollar variable... - //or - //not exists(EssaVariable var | var.getAUse() = m.getANormalExit() and var.getSourceVariable().getName() = name) and - //exists(EssaVariable var, PointsToContext2 imp | - // var.getAUse() = m.getANormalExit() and isModuleStateVariable(var) | - // PointsTo2::ssa_variable_named_attribute_points_to(var, imp, name, obj, origin) and - // imp.isImport() and obj != ObjectInternal::undefined() - //) - } - - /** Holds if `package.name` points to `(value, origin)`, where `package` is a package object. */ - cached predicate package_attribute_points_to(PackageObjectInternal package, string name, ObjectInternal value, CfgOrigin origin) { - py_module_attributes(package.getInitModule().getSourceModule(), name, value, origin) - or - // TO DO - //exists(Module init | - // init = package.getInitModule().getModule() and - // not exists(EssaVariable var | var.getAUse() = init.getANormalExit() and var.getSourceVariable().getName() = name) and - // exists(EssaVariable var, Context context | - // isModuleStateVariable(var) and var.getAUse() = init.getANormalExit() and - // context.isImport() and - // SSA::ssa_variable_named_attribute_points_to(var, context, name, undefinedVariable(), _, _) and - // origin = value and - // value = package.submodule(name) - // ) - //) - //or - package.hasNoInitModule() and - exists(ModuleObjectInternal mod | - mod = package.submodule(name) and - value = mod | - origin = CfgOrigin::fromModule(mod) - ) - } - /** Implicit "definition" of the names of submodules at the start of an `__init__.py` file. * * PointsTo isn't exactly how the interpreter works, but is the best approximation we can manage statically. @@ -761,15 +726,6 @@ private predicate potential_builtin_points_to(NameNode f, ObjectInternal value, ) } -/** Get the ESSA pseudo-variable used to retain module state - * during module initialization. Module attributes are handled - * as attributes of this variable, allowing the SSA form to track - * mutations of the module during its creation. - */ -private predicate isModuleStateVariable(EssaVariable var) { - var.getName() = "$" and var.getScope() instanceof Module -} - module Conditionals { /** Holds if `expr` is the operand of a unary `not` expression. */ @@ -920,7 +876,7 @@ module Types { } predicate declaredAttribute(ClassObjectInternal cls, string name, ObjectInternal value, CfgOrigin origin) { - value.getBuiltin() = cls.getBuiltin().getMember(name) and origin = CfgOrigin::unknown() + value = ObjectInternal::fromBuiltin(cls.getBuiltin().getMember(name)) and origin = CfgOrigin::unknown() or value != ObjectInternal::undefined() and exists(EssaVariable var | @@ -944,12 +900,6 @@ module Types { obj = ObjectInternal::unknown() and result = ObjectInternal::unknownClass() ) or - exists(Builtin meta | - result.getBuiltin() = meta and - meta = cls.getBuiltin().getClass() and - meta.inheritsFromType() - ) - or exists(ControlFlowNode meta | six_add_metaclass(_, cls, meta) and PointsTo2::points_to(meta, _, result, _) diff --git a/python/ql/src/semmle/python/types/Builtins.qll b/python/ql/src/semmle/python/types/Builtins.qll index f5f0756457a2..a61d7f9e4ad1 100644 --- a/python/ql/src/semmle/python/types/Builtins.qll +++ b/python/ql/src/semmle/python/types/Builtins.qll @@ -81,6 +81,22 @@ class Builtin extends @py_cobject { this.getClass().getName() = "wrapper_descriptor" } + int intValue() { + (this.getClass() = Builtin::special("int") or + this.getClass() = Builtin::special("long")) and + result = this.getName().toInt() + } + + string strValue() { + (this.getClass() = Builtin::special("unicode") or + this.getClass() = Builtin::special("bytes")) and + exists(string quoted_string | + quoted_string = this.getName() + and + result = quoted_string.regexpCapture("[bu]'([\\s\\S]*)'", 1) + ) + } + } module Builtin { From e3ed8c6abfa69153c0de878612a2d15a3029c592 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 21 Mar 2019 13:54:18 +0000 Subject: [PATCH 009/128] Python points-to: Simplify handling of booleans and comparisons. --- .../src/semmle/python/objects/Callables.qll | 3 - .../ql/src/semmle/python/objects/Classes.qll | 2 - .../src/semmle/python/objects/Constants.qll | 12 +- .../src/semmle/python/objects/Instances.qll | 22 +--- .../ql/src/semmle/python/objects/Modules.qll | 29 +---- .../src/semmle/python/objects/ObjectAPI.qll | 15 ++- .../semmle/python/objects/ObjectInternal.qll | 29 ++--- .../ql/src/semmle/python/objects/TObject.qll | 4 +- .../src/semmle/python/pointsto/PointsTo2.qll | 115 +++++------------- .../PointsTo/comparisons/PointsTo.expected | 27 ++++ .../PointsTo/comparisons/PointsTo.ql | 10 ++ .../PointsTo/comparisons/test.py | 38 ++++++ 12 files changed, 139 insertions(+), 167 deletions(-) create mode 100644 python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected create mode 100644 python/ql/test/library-tests/PointsTo/comparisons/PointsTo.ql create mode 100644 python/ql/test/library-tests/PointsTo/comparisons/test.py diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 1b95813cd9ca..b26103b1089c 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -26,9 +26,6 @@ abstract class CallableObjectInternal extends ObjectInternal { result = true } - /** Holds if this object may be true or false when evaluated as a bool */ - override predicate maybe() { none() } - override ClassDecl getClassDeclaration() { none() } diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 1d0ec77301ed..473517ba6af2 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -15,8 +15,6 @@ abstract class ClassObjectInternal extends ObjectInternal { result = true } - override predicate maybe() { none() } - override boolean isClass() { result = true } override int intValue() { diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index 7a9315fc210e..2874418153fa 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -13,8 +13,6 @@ abstract class BooleanObjectInternal extends ObjectInternal { this = TTrue() or this = TFalse() } - override predicate maybe() { none() } - override ClassDecl getClassDeclaration() { none() } @@ -111,8 +109,6 @@ class NoneObjectInternal extends ObjectInternal, TNone { result = false } - override predicate maybe() { none() } - override ClassDecl getClassDeclaration() { none() } @@ -175,8 +171,6 @@ class IntObjectInternal extends ObjectInternal, TInt { node.getNode().(IntegerLiteral).getValue() = this.intValue() } - override predicate maybe() { none() } - override ClassDecl getClassDeclaration() { none() } @@ -233,7 +227,7 @@ class IntObjectInternal extends ObjectInternal, TInt { class StringObjectInternal extends ObjectInternal, TString { override string toString() { - result = "'" + this.strValue().toString() + "'" + result = "'" + this.strValue() + "'" } override predicate introduced(ControlFlowNode node, PointsToContext2 context) { @@ -241,8 +235,6 @@ class StringObjectInternal extends ObjectInternal, TString { node.getNode().(StrConst).getText() = this.strValue() } - override predicate maybe() { none() } - override ClassDecl getClassDeclaration() { none() } @@ -252,7 +244,7 @@ class StringObjectInternal extends ObjectInternal, TString { override boolean isComparable() { result = true } override ObjectInternal getClass() { - result = TBuiltinClassObject(Builtin::special("str")) + result = TBuiltinClassObject(Builtin::special("unicode")) } override Builtin getBuiltin() { diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 4c2df644d5e1..c01c0a993ef2 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -10,19 +10,13 @@ private import semmle.python.types.Builtins class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { override string toString() { - result = "instance of " + this.getClass().(ClassObjectInternal).getName() + result = this.getOrigin().toString() } /** The boolean value of this object, if it has one */ override boolean booleanValue() { - //this.getClass().instancesAlways(result) - none() - } - - /** Holds if this object may be true or false when evaluated as a bool */ - override predicate maybe() { - // this.getClass().instancesMaybe() - any() + //result = this.getClass().instancesBooleanValue() + result = maybe() } override predicate introduced(ControlFlowNode node, PointsToContext2 context) { @@ -97,14 +91,8 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { /** The boolean value of this object, if it has one */ override boolean booleanValue() { - //this.getClass().instancesAlways(result) - none() - } - - /** Holds if this object may be true or false when evaluated as a bool */ - override predicate maybe() { - // this.getClass().instancesMaybe() - any() + //result = this.getClass().instancesBooleanValue() + result = maybe() } override predicate introduced(ControlFlowNode node, PointsToContext2 context) { diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 0e6529103587..636e2af2d887 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -25,6 +25,10 @@ abstract class ModuleObjectInternal extends ObjectInternal { override boolean isComparable() { result = true } + override boolean booleanValue() { + result = true + } + } class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleObject { @@ -45,12 +49,6 @@ class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleOb none() } - override boolean booleanValue() { - result = true - } - - override predicate maybe() { none() } - override ClassDecl getClassDeclaration() { none() } @@ -106,12 +104,6 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { none() } - override boolean booleanValue() { - result = true - } - - override predicate maybe() { none() } - override ClassDecl getClassDeclaration() { none() } @@ -139,13 +131,6 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { result.getName() = this.getName() + "." + name } - override predicate hasLocationInfo(string fp, int bl, int bc, int el, int ec) { - this.getOrigin().getLocation().hasLocationInfo(fp, bl, bc, el, ec) - or - this.hasNoInitModule() and fp = this.getFolder().getAbsolutePath() and - bl = 0 and bc = 0 and el = 0 and ec = 0 - } - override int intValue() { none() } @@ -212,12 +197,6 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { none() } - override boolean booleanValue() { - result = true - } - - override predicate maybe() { none() } - override ClassDecl getClassDeclaration() { none() } diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 77af652dd9c6..11850164117b 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -1,6 +1,6 @@ import python private import TObject -private import ObjectInternal +private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo2 private import semmle.python.pointsto.PointsToContext2 @@ -22,4 +22,17 @@ class Value extends TObject { result = this.(ObjectInternal).getClass() } + CallNode getACall() { + PointsTo2::points_to(result.getFunction(), _, this, _) + or + exists(BoundMethodObjectInternal bm | + PointsTo2::points_to(result.getFunction(), _, bm, _) and + bm.getFunction() = this + ) + } + + Value attr(string name) { + this.(ObjectInternal).attribute(name, result, _) + } + } diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index b9e3153527ef..c4791a039c86 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -14,12 +14,10 @@ class ObjectInternal extends TObject { abstract string toString(); - /** The boolean value of this object, if it has one */ + /** The boolean value of this object, this may be both + * true and false if the "object" represents a set of possible objects. */ abstract boolean booleanValue(); - /** Holds if this object may be true or false when evaluated as a bool */ - abstract predicate maybe(); - abstract predicate introduced(ControlFlowNode node, PointsToContext2 context); /** Gets the class declaration for this object, if it is a declared class. */ @@ -54,10 +52,6 @@ class ObjectInternal extends TObject { */ abstract predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin); - predicate hasLocationInfo(string fp, int bl, int bc, int el, int ec) { - this.getOrigin().getLocation().hasLocationInfo(fp, bl, bc, el, ec) - } - /** The integer value of things that have integer values. * That is, ints and bools. */ @@ -92,13 +86,8 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { } override boolean booleanValue() { - // TO DO ... Depends on class. `this.getClass().instancesAlways(result)` - none() - } - - override predicate maybe() { - // TO DO ... Depends on class. `this.getClass().instancesMaybe()` - any() + // TO DO ... Depends on class. `result = this.getClass().instancesBooleanValue()` + result = maybe() } override ClassDecl getClassDeclaration() { @@ -155,11 +144,9 @@ class UnknownInternal extends ObjectInternal, TUnknown { } override boolean booleanValue() { - none() + result = maybe() } - override predicate maybe() { any() } - override ClassDecl getClassDeclaration() { none() } @@ -219,8 +206,6 @@ class UndefinedInternal extends ObjectInternal, TUndefined { none() } - override predicate maybe() { none() } - override ClassDecl getClassDeclaration() { none() } @@ -328,3 +313,7 @@ module ObjectInternal { } } +/** Helper for boolean predicates returning both `true` and `false` */ +boolean maybe() { + result = true or result = false +} diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 29957f87e0c0..3e7815e5b3bf 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -64,9 +64,7 @@ newtype TObject = or TString(string s) { // Any string explicitly mentioned in the source code. - exists(StrConst str | - s = str.getText() - ) + s = any(StrConst str).getText() or // Any string from the library put in the DB by the extractor. s = any(Builtin b).strValue() diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index 183dd9df917d..0dc17dc16d7c 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -139,13 +139,8 @@ module PointsTo2 { reachableBlock(guard, context) | exists(ObjectInternal value | - points_to(guard.getLastNode(), context, value, _) - | - guard.controls(b, _) and value.maybe() - or - guard.controls(b, true) and value.booleanValue() = true - or - guard.controls(b, false) and value.booleanValue() = false + points_to(guard.getLastNode(), context, value, _) and + guard.controls(b, value.booleanValue()) ) or /* Assume the true edge of an assert is reachable (except for assert 0/False) */ @@ -162,13 +157,8 @@ module PointsTo2 { */ predicate controlledReachableEdge(BasicBlock pred, BasicBlock succ, PointsToContext2 context) { exists(ConditionBlock guard, ObjectInternal value | - points_to(guard.getLastNode(), context, value, _) - | - guard.controlsEdge(pred, succ, _) and value.maybe() - or - guard.controlsEdge(pred, succ, true) and value.booleanValue() = true - or - guard.controlsEdge(pred, succ, false) and value.booleanValue() = false + points_to(guard.getLastNode(), context, value, _) and + guard.controlsEdge(pred, succ, value.booleanValue()) ) } @@ -324,7 +314,7 @@ module PointsTo2 { /** Holds if ESSA edge refinement, `def`, refers to `(value, cls, origin)`. */ predicate ssa_filter_definition_points_to(PyEdgeRefinement def, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { exists(ControlFlowNode test, ControlFlowNode use | - refinement_test(test, use, Conditionals::evaluatesTo(test, use, context, value, origin.toCfgNode()), def) + refinement_test(test, use, Conditionals::branchEvaluatesTo(test, use, context, value, origin.toCfgNode()), def) ) } @@ -337,7 +327,7 @@ module PointsTo2 { */ use = uniphi.getInput().getSourceVariable().(Variable).getAUse() and test = uniphi.getDefiningNode() and - uniphi.getSense() = Conditionals::evaluatesTo(test, use, context, value, origin.toCfgNode()) + uniphi.getSense() = Conditionals::branchEvaluatesTo(test, use, context, value, origin.toCfgNode()) ) } @@ -439,41 +429,9 @@ module PointsTo2 { private predicate compare_expr_points_to(CompareNode cmp, PointsToContext2 context, ObjectInternal value) { - exists(ControlFlowNode a, ControlFlowNode b, ObjectInternal o1, ObjectInternal o2 | - exists(boolean is | - equality_test(cmp, a, is, b) and - points_to(a, context, o1, _) and - points_to(b, context, o2, _) | - (o1.isComparable() = true and o2.isComparable() = true) and - ( - o1 = o2 and value = ObjectInternal::bool(is) - or - o1 != o2 and value = ObjectInternal::bool(is.booleanNot()) - ) - or - (o1.isComparable() = false or o2.isComparable() = false) and - value = ObjectInternal::bool(_) - ) - or - exists(boolean strict | - inequality(cmp, a, b, strict) and - points_to(a, context, o1, _) and - points_to(b, context, o2, _) | - o1.intValue() < o2.intValue() and value = ObjectInternal::bool(true) - or - o1.intValue() > o2.intValue() and value = ObjectInternal::bool(false) - or - o1.intValue() = o2.intValue() and value = ObjectInternal::bool(strict.booleanNot()) - or - o1.strValue() < o2.strValue() and value = ObjectInternal::bool(true) - or - o1.strValue() > o2.strValue() and value = ObjectInternal::bool(false) - or - o1.strValue() = o2.strValue() and value = ObjectInternal::bool(strict.booleanNot()) - ) - // or - // value = version_tuple_compare(cmp, context) - ) + value = ObjectInternal::bool(Conditionals::comparisonEvaluatesTo(cmp, _, context, _, _)) + // or + // value = version_tuple_compare(cmp, context) } /** Helper for comparisons. */ @@ -494,11 +452,7 @@ module PointsTo2 { op = f.getNode().getOp() and points_to(f.getOperand(), context, operand, _) | - op instanceof Not and operand.maybe() and value = ObjectInternal::bool(_) - or - op instanceof Not and operand.booleanValue() = false and value = ObjectInternal::bool(true) - or - op instanceof Not and operand.booleanValue() = true and value = ObjectInternal::bool(false) + op instanceof Not and value = ObjectInternal::bool(operand.booleanValue().booleanNot()) or op instanceof USub and value = ObjectInternal::fromInt(-operand.intValue()) or @@ -734,48 +688,34 @@ module Conditionals { result = expr.(UnaryExprNode).getOperand() } - private boolean maybe() { - result = true or result = false + boolean branchEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { + contains_interesting_expression_within_test(expr, use) and + PointsTo2::points_to(use, context, val, origin) and + expr = use and + val.booleanValue() = result + or + result = comparisonEvaluatesTo(expr, use, context, val, origin) + or + result = branchEvaluatesTo(not_operand(expr), use, context, val, origin).booleanNot() } - boolean evaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { + boolean comparisonEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { + result = equalityEvaluatesTo(expr, use, context, val, origin) + or + result = inequalityEvaluatesTo(expr, use, context, val, origin) + // or //result = isinstance_test_evaluates_boolean(expr, use, context, val, origin) //or //result = issubclass_test_evaluates_boolean(expr, use, context, val, origin) //or - result = equalityEvaluatesTo(expr, use, context, val, origin) - or - result = inequalityEvaluatesTo(expr, use, context, val, origin) - or //result = callable_test_evaluates_boolean(expr, use, context, val, origin) //or //result = hasattr_test_evaluates_boolean(expr, use, context, val, origin) - //or - result = evaluatesTo(not_operand(expr), use, context, val, origin).booleanNot() - or - //result = true and evaluates_int(expr, use, context, val, origin) != 0 - //or - //result = false and evaluates_int(expr, use, context, val, origin) = 0 - //or - result = boolEvaluatesTo(expr, use, context, val, origin) - } - - pragma [noinline] - private boolean boolEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { - contains_interesting_expression_within_test(expr, use) and - PointsTo2::points_to(use, context, val, origin) and - expr = use and - ( - val.booleanValue() = result - or - val.maybe() and result = maybe() - ) } pragma [noinline] private boolean equalityEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { exists(ControlFlowNode r, boolean sense | - contains_interesting_expression_within_test(expr, use) | equality_test(expr, use, sense, r) and exists(ObjectInternal other | PointsTo2::points_to(use, context, val, origin) and @@ -797,7 +737,6 @@ module Conditionals { pragma [noinline] private boolean inequalityEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { exists(ControlFlowNode r, boolean sense | - contains_interesting_expression_within_test(expr, use) | exists(boolean strict, ObjectInternal other | ( PointsTo2::inequality(expr, use, r, strict) and sense = true @@ -805,7 +744,7 @@ module Conditionals { PointsTo2::inequality(expr, r, use, strict) and sense = false ) and PointsTo2::points_to(use, context, val, origin) and - PointsTo2::points_to(r, context, other, _) + PointsTo2::points_to(r, context, other, _) | val.intValue() < other.intValue() and result = sense or @@ -818,6 +757,10 @@ module Conditionals { val.strValue() > other.strValue() and result = sense.booleanNot() or val.strValue() = other.strValue() and result = strict.booleanXor(sense) + or + val.isComparable() = false and result = maybe() + or + other.isComparable() = false and result = maybe() ) ) diff --git a/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected b/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected new file mode 100644 index 000000000000..8387ec955739 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected @@ -0,0 +1,27 @@ +| 2 | True | +| 3 | False | +| 6 | True | +| 7 | True | +| 8 | True | +| 9 | False | +| 19 | False | +| 19 | True | +| 20 | False | +| 20 | True | +| 21 | False | +| 21 | True | +| 22 | False | +| 22 | True | +| 25 | False | +| 26 | False | +| 27 | True | +| 28 | True | +| 29 | False | +| 30 | True | +| 33 | False | +| 33 | True | +| 34 | True | +| 35 | False | +| 36 | False | +| 37 | True | +| 38 | True | diff --git a/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.ql b/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.ql new file mode 100644 index 000000000000..c5bf9286904a --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.ql @@ -0,0 +1,10 @@ + +import python +import semmle.python.objects.ObjectAPI + +from int line, ControlFlowNode f, Value v +where + any(ExprStmt s).getValue() = f.getNode() and + line = f.getLocation().getStartLine() and + f = v.getAReferent() +select line, v diff --git a/python/ql/test/library-tests/PointsTo/comparisons/test.py b/python/ql/test/library-tests/PointsTo/comparisons/test.py new file mode 100644 index 000000000000..cc6010e7a39b --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/comparisons/test.py @@ -0,0 +1,38 @@ +#Strings +"a" < "b" +"aa" < "a" + +#Integers +-4 < 0 +0 > -3 +0 < 1 +1 == 5 + +#Too large +984523954611637034671 < 5857262868067092615796134571 + +#Nonsense +1 < "a" +len(unknown()) = "foo" + +#Imprecise +len(unknown()) == 5 +len(unknown()) < 7 +len(unknown()) == len(unknown()) +len(unknown()) < len(unknown()) + +#All ops +2 > 3 +2 == 3 +2 != 3 +2 < 3 +2 >= 3 +2 <= 3 + +#Booleans +1 < True +2 > False +True == False +True is False +True is True +False is not True From aa3074549225130373da4f53cb3dcdac779568c6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 21 Mar 2019 19:33:17 +0000 Subject: [PATCH 010/128] Python points-to: Further types and flow. --- .../src/semmle/python/objects/Callables.qll | 23 +- .../ql/src/semmle/python/objects/Classes.qll | 19 +- .../src/semmle/python/objects/Constants.qll | 28 +- .../src/semmle/python/objects/Instances.qll | 80 +++++- .../ql/src/semmle/python/objects/Modules.qll | 11 +- .../semmle/python/objects/ObjectInternal.qll | 35 ++- .../ql/src/semmle/python/objects/TObject.qll | 129 ++++++++- .../src/semmle/python/pointsto/PointsTo2.qll | 261 +++++++++++++++--- 8 files changed, 507 insertions(+), 79 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index b26103b1089c..2a12f0a20cda 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -81,6 +81,10 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti ) } + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + none() + } + override predicate calleeAndOffset(Function scope, int paramOffset) { scope = this.getScope() and paramOffset = 0 } @@ -98,7 +102,7 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc } override string toString() { - result = "builtin function " + this.getBuiltin().getName() + result = "Builtin-function " + this.getBuiltin().getName() } override predicate introduced(ControlFlowNode node, PointsToContext2 context) { @@ -112,6 +116,10 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc override boolean isComparable() { result = true } override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { exists(Builtin func, ClassObjectInternal cls | func = this.getBuiltin() and func != Builtin::builtin("isinstance") and @@ -121,8 +129,7 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc cls = ObjectInternal::fromBuiltin(this.getReturnType()) and obj = TUnknownInstance(cls) ) and - origin = CfgOrigin::unknown() and - callee_for_object(callee, this) + origin = CfgOrigin::unknown() } override ControlFlowNode getOrigin() { @@ -185,6 +192,10 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod override boolean isComparable() { result = true } override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. none() } @@ -235,8 +246,12 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { this.getFunction().callResult(callee, obj, origin) } + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + this.getFunction().callResult(obj, origin) + } + override ControlFlowNode getOrigin() { - this = TBoundMethod(result, _, _, _) + none() } override predicate calleeAndOffset(Function scope, int paramOffset) { diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 473517ba6af2..6aa308ea9a17 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -86,7 +86,11 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject override predicate attributesUnknown() { none() } override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - // TO DO .. Result should (in most cases) be an instance + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + // Handled by Instance classes. none() } @@ -101,7 +105,7 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec } override string toString() { - result = "builtin class " + this.getBuiltin().getName() + result = "builtin-class " + this.getBuiltin().getName() } override predicate introduced(ControlFlowNode node, PointsToContext2 context) { @@ -132,6 +136,10 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec override predicate attributesUnknown() { none() } override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { // TO DO .. Result should (in most cases) be an instance none() } @@ -166,8 +174,11 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { } override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and - callee_for_object(callee, this) + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() } override ControlFlowNode getOrigin() { diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index 2874418153fa..c8deab2657eb 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -35,6 +35,11 @@ abstract class BooleanObjectInternal extends ObjectInternal { none() } + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + // Booleans aren't callable + none() + } + override ControlFlowNode getOrigin() { none() } @@ -54,7 +59,7 @@ abstract class BooleanObjectInternal extends ObjectInternal { class TrueObjectInternal extends BooleanObjectInternal, TTrue { override string toString() { - result = "True" + result = "bool True" } override boolean booleanValue() { @@ -78,7 +83,7 @@ class TrueObjectInternal extends BooleanObjectInternal, TTrue { class FalseObjectInternal extends BooleanObjectInternal, TFalse { override string toString() { - result = "False" + result = "bool False" } override boolean booleanValue() { @@ -134,7 +139,12 @@ class NoneObjectInternal extends ObjectInternal, TNone { // None isn't callable none() } - + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + // None isn't callable + none() + } + override ControlFlowNode getOrigin() { none() } @@ -163,7 +173,7 @@ class NoneObjectInternal extends ObjectInternal, TNone { class IntObjectInternal extends ObjectInternal, TInt { override string toString() { - result = this.intValue().toString() + result = "int " + this.intValue().toString() } override predicate introduced(ControlFlowNode node, PointsToContext2 context) { @@ -193,6 +203,11 @@ class IntObjectInternal extends ObjectInternal, TInt { none() } + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + // ints aren't callable + none() + } + override ControlFlowNode getOrigin() { none() } @@ -256,6 +271,11 @@ class StringObjectInternal extends ObjectInternal, TString { none() } + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + // strings aren't callable + none() + } + override ControlFlowNode getOrigin() { none() } diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index c01c0a993ef2..596769d4dcc0 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -10,7 +10,7 @@ private import semmle.python.types.Builtins class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { override string toString() { - result = this.getOrigin().toString() + result = this.getOrigin().getNode().toString() } /** The boolean value of this object, if it has one */ @@ -53,16 +53,17 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { this = TSpecificInstance(result, _, _) } - /** Holds if `obj` is the result of calling `this` and `origin` is - * the origin of `obj`. - */ override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { // In general instances aren't callable, but some are... // TO DO -- Handle cases where class overrides __call__ none() } - override int intValue() { + override int intValue() { none() } @@ -96,7 +97,9 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { } override predicate introduced(ControlFlowNode node, PointsToContext2 context) { - none() + context.appliesTo(node) and + this.getClass() = ObjectInternal::builtin("float") and + node.getNode() instanceof FloatLiteral } /** Gets the class declaration for this object, if it is a declared class. */ @@ -129,10 +132,11 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { none() } - /** Holds if `obj` is the result of calling `this` and `origin` is - * the origin of `obj`. - */ override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { // In general instances aren't callable, but some are... // TO DO -- Handle cases where class overrides __call__ none() @@ -157,3 +161,61 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { override predicate attributesUnknown() { any() } } + + +class SuperInstance extends TSuperInstance, ObjectInternal { + + override string toString() { + result = "super()" + } + + override boolean booleanValue() { result = true } + + override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + exists(ObjectInternal self, ClassObjectInternal startclass | + super_instantiation(node, self, startclass, context) and + this = TSuperInstance(self, startclass) + ) + } + + ClassObjectInternal getStartClass() { + this = TSuperInstance(_, result) + } + + ObjectInternal getSelf() { + this = TSuperInstance(result, _) + } + + override ClassDecl getClassDeclaration() { none() } + + override boolean isClass() { result = false } + + override ObjectInternal getClass() { none() } + + override boolean isComparable() { none() } + + override Builtin getBuiltin() { none() } + + override ControlFlowNode getOrigin() { + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } + + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { none() } + + override int intValue() { none() } + + override string strValue() { none() } + + override predicate calleeAndOffset(Function scope, int paramOffset) { none() } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate attributesUnknown() { none() } + +} + + + + diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 636e2af2d887..920f9e45b6e5 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -17,6 +17,11 @@ abstract class ModuleObjectInternal extends ObjectInternal { none() } + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + // Modules aren't callable + none() + } + override ControlFlowNode getOrigin() { result = this.getSourceModule().getEntryNode() } @@ -38,7 +43,7 @@ class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleOb } override string toString() { - result = "builtin module " + this.getBuiltin().getName() + result = "Module " + this.getBuiltin().getName() } override string getName() { @@ -93,7 +98,7 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { } override string toString() { - result = "package " + this.getName() + result = "Package " + this.getName() } override string getName() { @@ -186,7 +191,7 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { } override string toString() { - result = "package " + this.getName() + result = "Module " + this.getName() } override string getName() { diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index c4791a039c86..259635d81de0 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -50,6 +50,11 @@ class ObjectInternal extends TObject { /** Holds if `obj` is the result of calling `this` and `origin` is * the origin of `obj`. */ + abstract predicate callResult(ObjectInternal obj, CfgOrigin origin); + + /** Holds if `obj` is the result of calling `this` and `origin` is + * the origin of `obj` with callee context `callee`. + */ abstract predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin); /** The integer value of things that have integer values. @@ -107,8 +112,11 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { override boolean isComparable() { result = false } override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and - callee_for_object(callee, this) + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() } override ControlFlowNode getOrigin() { @@ -167,9 +175,12 @@ class UnknownInternal extends ObjectInternal, TUnknown { none() } + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() + } + override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { - obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and - callee_for_object(callee, this) + none() } override ControlFlowNode getOrigin() { @@ -227,10 +238,13 @@ class UndefinedInternal extends ObjectInternal, TUndefined { } override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { // Accessing an undefined value raises a NameError, but if during import it probably // means that we missed an import. - obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() and - callee.getOuter().isImport() + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() } override ControlFlowNode getOrigin() { @@ -311,6 +325,15 @@ module ObjectInternal { or result = TBuiltinModuleObject(b) } + + ObjectInternal classMethod() { + result = TBuiltinClassObject(Builtin::special("ClassMethod")) + } + + ObjectInternal staticMethod() { + result = TBuiltinClassObject(Builtin::special("StaticMethod")) + } + } /** Helper for boolean predicates returning both `true` and `false` */ diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 3e7815e5b3bf..1f2433fea901 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -72,26 +72,133 @@ newtype TObject = s = "__main__" } or - TSpecificInstance(CallNode instantiation, ClassObjectInternal cls, PointsToContext2 context) { - PointsTo2::points_to(instantiation.getFunction(), context, cls, _) and + TSpecificInstance(ControlFlowNode instantiation, ClassObjectInternal cls, PointsToContext2 context) { + PointsTo2::points_to(instantiation.(CallNode).getFunction(), context, cls, _) and cls.isSpecial() = false + or + // Self + self_parameter(instantiation.getNode(), context, cls) } or TBoundMethod(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext2 context) { - exists(ControlFlowNode objnode, string name | - objnode = instantiation.getObject(name) and - PointsTo2::points_to(objnode, context, self, _) and - self.getClass().(ClassObjectInternal).attribute(name, function, _) - ) + method_binding(instantiation, self, function, context) } or TUnknownInstance(ClassObjectInternal cls) { cls != TUnknownClass() } + or + TSuperInstance(ObjectInternal self, ClassObjectInternal startclass) { + super_instantiation(_, self, startclass, _) + } private predicate is_power_2(int n) { n = 1 or exists(int half | is_power_2(half) and n = half*2) } +predicate super_instantiation(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext2 context) { + PointsTo2::points_to(instantiation.getFunction(), context, ObjectInternal::builtin("super"), _) and + ( + PointsTo2::points_to(instantiation.getArg(0), context, startclass, _) and + PointsTo2::points_to(instantiation.getArg(1), context, self, _) + or + major_version() = 3 and + not exists(instantiation.getArg(0)) and + exists(Function func | + instantiation.getScope() = func and + /* Implicit class argument is lexically enclosing scope */ + func.getScope() = startclass.(PythonClassObjectInternal).getScope() and + /* Implicit 'self' is the 0th parameter */ + PointsTo2::points_to(func.getArg(0).asName().getAFlowNode(), context, self, _) + ) + ) +} + +predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext2 context) { + exists(ObjectInternal obj, string name | + receiver(instantiation, context, obj, name) | + exists(ObjectInternal cls | + cls = obj.getClass() and + cls != ObjectInternal::builtin("super") and + cls.attribute(name, function, _) and + self = obj + ) + or + exists(SuperInstance sup | + sup = obj and + sup.getStartClass().attribute(name, function, _) and + self = sup.getSelf() + ) + ) +} + + +/** Helper for method_binding */ +pragma [noinline] +predicate receiver(AttrNode instantiation, PointsToContext2 context, ObjectInternal obj, string name) { + PointsTo2::points_to(instantiation.getObject(name), context, obj, _) +} + +/** Helper self parameters: `def meth(self, ...): ...`. */ +pragma [noinline] +private predicate self_parameter(Parameter def, PointsToContext2 context, PythonClassObjectInternal cls) { + def.isSelf() and + exists(Function scope | + def.(Name).getScope() = scope and + def.isSelf() and + context.isRuntime() and context.appliesToScope(scope) and + scope.getScope() = cls.getScope() and + concrete_class(cls) and + /* We want to allow decorated functions, otherwise we lose a lot of useful information. + * However, we want to exclude any function whose arguments are permuted by the decorator. + * In general we can't do that, but we can special case the most common ones. + */ + neither_class_nor_static_method(scope) + ) +} + +/** INTERNAL -- Use `not cls.isAbstract()` instead. */ +cached predicate concrete_class(PythonClassObjectInternal cls) { + cls.getClass() != abcMetaClassObject() + or + exists(Class c | + c = cls.getScope() and + not exists(c.getMetaClass()) + | + forall(Function f | + f.getScope() = c | + not exists(Raise r, Name ex | + r.getScope() = f and + (r.getException() = ex or r.getException().(Call).getFunc() = ex) and + (ex.getId() = "NotImplementedError" or ex.getId() = "NotImplemented") + ) + ) + ) +} + +private PythonClassObjectInternal abcMetaClassObject() { + /* Avoid using points-to and thus negative recursion */ + exists(Class abcmeta | + result.getScope() = abcmeta | + abcmeta.getName() = "ABCMeta" and + abcmeta.getScope().getName() = "abc" + ) +} + +private predicate neither_class_nor_static_method(Function f) { + not exists(f.getADecorator()) + or + exists(ControlFlowNode deco | + deco = f.getADecorator().getAFlowNode() | + exists(ObjectInternal o | + PointsTo2::points_to(deco, _, o, _) | + o != ObjectInternal::staticMethod() and + o != ObjectInternal::classMethod() + ) + or not deco instanceof NameNode + ) +} + + library class ClassDecl extends @py_object { @@ -135,11 +242,3 @@ library class ClassDecl extends @py_object { } - -predicate callee_for_object(PointsToContext2 callee, ObjectInternal obj) { - exists(CallNode call, PointsToContext2 caller | - callee.fromCall(call, caller) and - PointsTo2::points_to(call.getFunction(), caller, obj, _) - ) -} - diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index 0dc17dc16d7c..4c1aaf0f3554 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -104,6 +104,9 @@ module PointsTo2 { predicate points_to_candidate(ControlFlowNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { use_points_to(f, context, value, origin) or + /* Not necessary, but for backwards compatibility */ + def_points_to(f, context, value, origin) + or attribute_load_points_to(f, context, value, origin) or subscript_points_to(f, context, value, origin) @@ -138,10 +141,7 @@ module PointsTo2 { guard = b.getImmediatelyControllingBlock() and reachableBlock(guard, context) | - exists(ObjectInternal value | - points_to(guard.getLastNode(), context, value, _) and - guard.controls(b, value.booleanValue()) - ) + allowsFlow(guard, b, context) or /* Assume the true edge of an assert is reachable (except for assert 0/False) */ guard.controls(b, true) and @@ -153,16 +153,30 @@ module PointsTo2 { ) } + pragma [noopt] + private predicate allowsFlow(ConditionBlock guard, BasicBlock b, PointsToContext2 context) { + exists(ObjectInternal value, boolean sense, ControlFlowNode test | + test = guard.getLastNode() and + points_to(test, context, value, _) and + sense = value.booleanValue() and + guard.controls(b, sense) + ) + } + /* Holds if the edge `pred` -> `succ` is reachable, given the context `context`. */ + pragma [noopt] predicate controlledReachableEdge(BasicBlock pred, BasicBlock succ, PointsToContext2 context) { - exists(ConditionBlock guard, ObjectInternal value | - points_to(guard.getLastNode(), context, value, _) and - guard.controlsEdge(pred, succ, value.booleanValue()) + exists(ConditionBlock guard, ObjectInternal value, boolean sense, ControlFlowNode test | + test = guard.getLastNode() and + points_to(test, context, value, _) and + sense = value.booleanValue() and + guard.controlsEdge(pred, succ, sense) ) } /** Gets an object pointed to by a use (of a variable). */ + pragma [noinline] private predicate use_points_to(NameNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { exists(CfgOrigin origin_or_obj | value != ObjectInternal::undefined() and @@ -171,6 +185,13 @@ module PointsTo2 { ) } + /** Gets an object pointed to by the definition of an ESSA variable. */ + pragma [noinline] + private predicate def_points_to(DefinitionNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + points_to(f.getValue(), context, value, origin) + } + + pragma [noinline] private predicate use_points_to_maybe_origin(NameNode f, PointsToContext2 context, ObjectInternal value, CfgOrigin origin_or_obj) { ssa_variable_points_to(fast_local_variable(f), context, value, origin_or_obj) or @@ -181,10 +202,12 @@ module PointsTo2 { } /** Holds if `var` refers to `(value, origin)` given the context `context`. */ + pragma [noinline] predicate ssa_variable_points_to(EssaVariable var, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { ssa_definition_points_to(var.getDefinition(), context, value, origin) } + pragma [noinline] private predicate name_lookup_points_to_maybe_origin(NameNode f, PointsToContext2 context, ObjectInternal value, CfgOrigin origin_or_obj) { exists(EssaVariable var | var = name_local_variable(f) | ssa_variable_points_to(var, context, value, origin_or_obj) @@ -199,6 +222,7 @@ module PointsTo2 { ssa_variable_points_to(name_local_variable(f), context, ObjectInternal::undefined(), _) } + pragma [noinline] private predicate global_lookup_points_to_maybe_origin(NameNode f, PointsToContext2 context, ObjectInternal value, CfgOrigin origin_or_obj) { ssa_variable_points_to(global_variable(f), context, value, origin_or_obj) or @@ -235,13 +259,17 @@ module PointsTo2 { } /** Holds if `f` is an attribute `x.attr` and points to `(value, cls, origin)`. */ + pragma [noinline] private predicate attribute_load_points_to(AttrNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { exists(ObjectInternal object, string name, CfgOrigin orig | points_to(f.getObject(name), context, object, _) | object.attribute(name, value, orig) and origin = orig.fix(f) - or - object.attributesUnknown() and origin = f and value = ObjectInternal::unknown() + ) + or + exists(ObjectInternal object | + points_to(f.getObject(), context, object, _) and + origin = f and value = ObjectInternal::unknown() ) // TO DO -- Support CustomPointsToAttribute //or @@ -298,12 +326,12 @@ module PointsTo2 { private predicate ssa_node_refinement_points_to(EssaNodeRefinement def, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { //method_callsite_points_to(def, context, value, origin) //or - //import_star_points_to(def, context, value, origin) - //or + InterModulePointsTo::import_star_points_to(def, context, value, origin) + or //attribute_assignment_points_to(def, context, value, origin) //or - //callsite_points_to(def, context, value, origin) - //or + InterProceduralPointsTo::callsite_points_to(def, context, value, origin) + or //argument_points_to(def, context, value, origin) //or //attribute_delete_points_to(def, context, value, origin) @@ -427,7 +455,7 @@ module PointsTo2 { none() } - + pragma [noinline] private predicate compare_expr_points_to(CompareNode cmp, PointsToContext2 context, ObjectInternal value) { value = ObjectInternal::bool(Conditionals::comparisonEvaluatesTo(cmp, _, context, _, _)) // or @@ -447,6 +475,7 @@ module PointsTo2 { ) } + pragma [noinline] private predicate unary_points_to(UnaryExprNode f, PointsToContext2 context, ObjectInternal value) { exists(Unaryop op, ObjectInternal operand | op = f.getNode().getOp() and @@ -464,6 +493,7 @@ module PointsTo2 { module InterModulePointsTo { + pragma [noinline] predicate import_points_to(ControlFlowNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { exists(string name, ImportExpr i | i.getAFlowNode() = f and i.getImportedModuleName() = name and @@ -474,10 +504,27 @@ module InterModulePointsTo { } predicate from_import_points_to(ImportMemberNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + from_self_import_points_to(f, context, value, origin) + or + from_other_import_points_to(f, context, value, origin) + } + + pragma [noinline] + predicate from_self_import_points_to(ImportMemberNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + exists(EssaVariable var, CfgOrigin orig | + var = ssa_variable_for_module_attribute(f, context) and + PointsTo2::ssa_variable_points_to(var, context, value, orig) and + origin = orig.asCfgNodeOrHere(f) + ) + } + + pragma [noinline] + predicate from_other_import_points_to(ImportMemberNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { exists(string name, ModuleObjectInternal mod, CfgOrigin orig | - PointsTo2::points_to(f.getModule(name), context, mod, _) and + from_import_imports(f, context, mod, name) and + (mod.getSourceModule() != f.getEnclosingModule() or mod.isBuiltin()) and + mod.attribute(name, value, origin) and origin = orig.asCfgNodeOrHere(f) - | // TO DO... $ variables. //mod.getSourceModule() = f.getEnclosingModule() and //not exists(EssaVariable var | var.getSourceVariable().getName() = name and var.getAUse() = f) and @@ -485,18 +532,14 @@ module InterModulePointsTo { // isModuleStateVariable(dollar) and dollar.getAUse() = f and // SSA::ssa_variable_named_attribute_points_to(dollar, context, name, value, orig) //) - //or - (mod.getSourceModule() != f.getEnclosingModule() or mod.isBuiltin()) and - mod.attribute(name, value, origin) - ) - or - exists(EssaVariable var, CfgOrigin orig | - var = ssa_variable_for_module_attribute(f, context) and - PointsTo2::ssa_variable_points_to(var, context, value, orig) and - origin = orig.asCfgNodeOrHere(f) ) } + private predicate from_import_imports(ImportMemberNode f, PointsToContext2 context, ModuleObjectInternal mod, string name) { + PointsTo2::points_to(f.getModule(name), context, mod, _) + } + + pragma [noinline] private EssaVariable ssa_variable_for_module_attribute(ImportMemberNode f, PointsToContext2 context) { exists(string name, ModuleObjectInternal mod, Module m | mod.getSourceModule() = m and m = f.getEnclosingModule() and m = result.getScope() and @@ -538,16 +581,114 @@ module InterModulePointsTo { ) } + /** Points-to for `from ... import *`. */ + predicate import_star_points_to(ImportStarRefinement def, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + exists(CfgOrigin orig | + origin = orig.fix(def.getDefiningNode()) + | + exists(ModuleObjectInternal mod, string name | + PointsTo2::points_to(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) and + name = def.getSourceVariable().getName() | + /* Attribute from imported module */ + module_exports_boolean(mod, name) = true and + mod.attribute(name, value, origin) + ) + or + exists(EssaVariable var | + /* Retain value held before import */ + variable_not_redefined_by_import_star(var, context, def) and + PointsTo2::ssa_variable_points_to(var, context, value,orig) + ) + ) + } + + + /** Holds if `def` is technically a definition of `var`, but the `from ... import *` does not in fact define `var`. */ + cached predicate variable_not_redefined_by_import_star(EssaVariable var, PointsToContext2 context, ImportStarRefinement def) { + var = def.getInput() and + exists(ModuleObjectInternal mod | + PointsTo2::points_to(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) | + module_exports_boolean(mod, var.getSourceVariable().getName()) = false + or + exists(Module m, string name | + m = mod.getSourceModule() and name = var.getSourceVariable().getName() | + not m.declaredInAll(_) and name.charAt(0) = "_" + ) + ) + } + + private predicate importsByImportStar(ModuleObjectInternal mod, ModuleObjectInternal imported) { + exists(ImportStarNode isn | + PointsTo2::points_to(isn.getModule(), _, imported, _) and + isn.getScope() = mod.getSourceModule() + ) + or exists(ModuleObjectInternal mid | + importsByImportStar(mod, mid) and importsByImportStar(mid, imported) + ) + } + + private predicate ofInterestInModule(ModuleObjectInternal mod, string name) { + exists(ImportStarNode isn, Module m | + m = mod.getSourceModule() and + isn.getScope() = m and + exists(EssaVariable var | var.getAUse() = isn and var.getName() = name) + ) + } + + private predicate ofInterestInExports(ModuleObjectInternal mod, string name) { + exists(ModuleObjectInternal importer | + importsByImportStar(importer, mod) and + ofInterestInModule(importer, name) + ) + } + + private boolean module_exports_boolean(ModuleObjectInternal mod, string name) { + ofInterestInExports(mod, name) and + exists(Module src | + src = mod.getSourceModule() + | + if exists(SsaVariable var | name = var.getId() and var.getAUse() = src.getANormalExit()) then + result = true + else ( + exists(ImportStarNode isn, ModuleObjectInternal imported | + isn.getScope() = src and + PointsTo2::points_to(isn.getModule(), _, imported, _) and + result = module_exports_boolean(imported, name) + ) + or + not exists(ImportStarNode isn |isn.getScope() = src) and result = false + ) + ) + or + ofInterestInExports(mod, name) and + exists(Folder folder | + mod.(PackageObjectInternal).hasNoInitModule() and + folder = mod.(PackageObjectInternal).getFolder() | + if (exists(folder.getChildContainer(name)) or exists(folder.getFile(name + ".py"))) then + result = true + else + result = false + ) + or + name = "__name__" and result = true + } + } module InterProceduralPointsTo { + pragma [noinline] predicate call_points_to(CallNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { - exists(ObjectInternal func, PointsToContext2 callee, CfgOrigin resultOrigin | - callee.fromCall(f, context) and + exists(ObjectInternal func, CfgOrigin resultOrigin | PointsTo2::points_to(f.getFunction(), context, func, _) and - func.callResult(callee, value, resultOrigin) and origin = resultOrigin.fix(f) + | + exists(PointsToContext2 callee | + callee.fromCall(f, context) and + func.callResult(callee, value, resultOrigin) + ) + or + func.callResult(value, resultOrigin) ) } @@ -627,12 +768,11 @@ module InterProceduralPointsTo { /** Model the transfer of values at scope-entry points. Transfer from `(pred_var, pred_context)` to `(succ_def, succ_context)`. */ cached predicate scope_entry_value_transfer(EssaVariable pred_var, PointsToContext2 pred_context, ScopeEntryDefinition succ_def, PointsToContext2 succ_context) { scope_entry_value_transfer_from_earlier(pred_var, pred_context, succ_def, succ_context) - // TO DO... - //or - //callsite_entry_value_transfer(pred_var, pred_context, succ_def, succ_context) - //or - //pred_context.isImport() and pred_context = succ_context and - //class_entry_value_transfer(pred_var, succ_def) + or + callsite_entry_value_transfer(pred_var, pred_context, succ_def, succ_context) + or + pred_context.isImport() and pred_context = succ_context and + class_entry_value_transfer(pred_var, succ_def) } /** Helper for `scope_entry_value_transfer`. Transfer of values from a temporally earlier scope to later scope. @@ -667,6 +807,59 @@ module InterProceduralPointsTo { ) } + /** Helper for `scope_entry_value_transfer`. + * Transfer of values from the callsite to the callee, for enclosing variables, but not arguments/parameters. */ + pragma [noinline] + private predicate callsite_entry_value_transfer(EssaVariable caller_var, PointsToContext2 caller, ScopeEntryDefinition entry_def, PointsToContext2 callee) { + entry_def.getSourceVariable() = caller_var.getSourceVariable() and + callsite_calls_function(caller_var.getAUse(), caller, entry_def.getScope(), callee, _) + } + + /** Helper for `scope_entry_value_transfer`. */ + private predicate class_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition succ_def) { + exists(ImportTimeScope scope, ControlFlowNode class_def | + class_def = pred_var.getAUse() and + scope.entryEdge(class_def, succ_def.getDefiningNode()) and + pred_var.getSourceVariable() = succ_def.getSourceVariable() + ) + } + + /** Points-to for a variable (possibly) redefined by a call: + * `var = ...; foo(); use(var)` + * Where var may be redefined in call to `foo` if `var` escapes (is global or non-local). + */ + pragma [noinline] + predicate callsite_points_to(CallsiteRefinement def, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + exists(SsaSourceVariable srcvar | + srcvar = def.getSourceVariable() | + if srcvar instanceof EscapingAssignmentGlobalVariable then ( + /* If global variable can be reassigned, we need to track it through calls */ + exists(EssaVariable var, Function func, PointsToContext2 callee | + callsite_calls_function(def.getCall(), context, func, callee, _) and + var_at_exit(srcvar, func, var) and + PointsTo2::ssa_variable_points_to(var, callee, value, origin) + ) + or + exists(ObjectInternal callable | + PointsTo2::points_to(def.getCall().getFunction(), context, callable, _) and + exists(callable.getBuiltin()) and + PointsTo2::ssa_variable_points_to(def.getInput(), context, value, origin) + ) + ) else ( + /* Otherwise we can assume its value (but not those of its attributes or members) has not changed. */ + PointsTo2::ssa_variable_points_to(def.getInput(), context, value, origin) + ) + ) + } + + /* Helper for computing ESSA variables at scoepe exit. */ + private predicate var_at_exit(Variable var, Scope scope, EssaVariable evar) { + not var instanceof LocalVariable and + evar.getSourceVariable() = var and + evar.getScope() = scope and + BaseFlow::reaches_exit(evar) + } + } /** Gets the `value, origin` that `f` would refer to if it has not been assigned some other value */ From dd83149cc345eadb961e59068dbeb66e1921e096 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 22 Mar 2019 14:51:51 +0000 Subject: [PATCH 011/128] Python points-to: Port old API classes to use new points-to. --- python/ql/src/python.qll | 1 + python/ql/src/semmle/python/Exprs.qll | 43 +- python/ql/src/semmle/python/Flow.qll | 57 +- python/ql/src/semmle/python/Import.qll | 2 +- python/ql/src/semmle/python/Module.qll | 8 +- python/ql/src/semmle/python/SelfAttribute.qll | 1 - .../src/semmle/python/objects/Callables.qll | 115 +- .../ql/src/semmle/python/objects/Classes.qll | 16 +- .../src/semmle/python/objects/Constants.qll | 44 +- .../src/semmle/python/objects/Instances.qll | 18 +- .../ql/src/semmle/python/objects/Modules.qll | 40 +- .../src/semmle/python/objects/ObjectAPI.qll | 83 +- .../semmle/python/objects/ObjectInternal.qll | 39 +- .../ql/src/semmle/python/objects/TObject.qll | 38 +- python/ql/src/semmle/python/pointsto/MRO2.qll | 6 +- .../src/semmle/python/pointsto/PointsTo.qll | 3032 ----------------- .../src/semmle/python/pointsto/PointsTo2.qll | 223 +- .../python/pointsto/PointsToContext.qll | 30 +- .../python/pointsto/PointsToContext2.qll | 272 -- .../src/semmle/python/types/ClassObject.qll | 99 +- .../src/semmle/python/types/Descriptors.qll | 46 +- .../semmle/python/types/FunctionObject.qll | 35 +- .../src/semmle/python/types/ModuleObject.qll | 91 +- python/ql/src/semmle/python/types/Object.qll | 40 +- python/ql/src/semmle/python/types/Version.qll | 151 - 25 files changed, 632 insertions(+), 3898 deletions(-) delete mode 100644 python/ql/src/semmle/python/pointsto/PointsTo.qll mode change 100755 => 100644 python/ql/src/semmle/python/pointsto/PointsToContext.qll delete mode 100644 python/ql/src/semmle/python/pointsto/PointsToContext2.qll diff --git a/python/ql/src/python.qll b/python/ql/src/python.qll index cdf33c8019f9..ddfd05ee52ee 100644 --- a/python/ql/src/python.qll +++ b/python/ql/src/python.qll @@ -36,5 +36,6 @@ import semmle.dataflow.SSA import semmle.python.pointsto.Base import semmle.python.pointsto.Context import semmle.python.pointsto.CallGraph +import semmle.python.objects.ObjectAPI import site diff --git a/python/ql/src/semmle/python/Exprs.qll b/python/ql/src/semmle/python/Exprs.qll index 81d6989768f3..2af87d47d56a 100644 --- a/python/ql/src/semmle/python/Exprs.qll +++ b/python/ql/src/semmle/python/Exprs.qll @@ -1,5 +1,5 @@ import python -private import semmle.python.pointsto.PointsTo +private import semmle.python.pointsto.PointsTo2 /** An expression */ class Expr extends Expr_, AstNode { @@ -78,37 +78,42 @@ class Expr extends Expr_, AstNode { * NOTE: For complex dataflow, involving multiple stages of points-to analysis, it may be more precise to use * `ControlFlowNode.refersTo(...)` instead. */ - predicate refersTo(Object value, ClassObject cls, AstNode origin) { - not py_special_objects(cls, "_semmle_unknown_type") - and - not value = unknownValue() - and - PointsTo::points_to(this.getAFlowNode(), _, value, cls, origin.getAFlowNode()) + predicate refersTo(Object obj, ClassObject cls, AstNode origin) { + this.refersTo(_, obj, cls, origin) } /** Gets what this expression might "refer-to" in the given `context`. */ - predicate refersTo(Context context, Object value, ClassObject cls, AstNode origin) { - not py_special_objects(cls, "_semmle_unknown_type") - and - PointsTo::points_to(this.getAFlowNode(), context, value, cls, origin.getAFlowNode()) + predicate refersTo(Context context, Object obj, ClassObject cls, AstNode origin) { + exists(Value value, ControlFlowNode cfgorigin | + PointsTo2::points_to(this.getAFlowNode(), context, value, cfgorigin) and + origin.getAFlowNode() = cfgorigin and + cls = value.getClass().getSource() | + if exists(value.getSource()) then + obj = value.getSource() + else + obj = cfgorigin + ) } /** Whether this expression might "refer-to" to `value` which is from `origin` * Unlike `this.refersTo(value, _, origin)`, this predicate includes results * where the class cannot be inferred. */ - predicate refersTo(Object value, AstNode origin) { - PointsTo::points_to(this.getAFlowNode(), _, value, _, origin.getAFlowNode()) - and - not value = unknownValue() + predicate refersTo(Object obj, AstNode origin) { + exists(Value value, ControlFlowNode cfgorigin | + PointsTo2::points_to(this.getAFlowNode(), _, value, cfgorigin) and + origin.getAFlowNode() = cfgorigin and + if exists(value.getSource()) then + obj = value.getSource() + else + obj = cfgorigin + ) } /** Equivalent to `this.refersTo(value, _)` */ - predicate refersTo(Object value) { - PointsTo::points_to(this.getAFlowNode(), _, value, _, _) - and - not value = unknownValue() + predicate refersTo(Object obj) { + this.refersTo(obj, _) } } diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll index 2b663323a48e..2cd0b355ab9b 100755 --- a/python/ql/src/semmle/python/Flow.qll +++ b/python/ql/src/semmle/python/Flow.qll @@ -1,6 +1,6 @@ import python import semmle.python.flow.NameNode -private import semmle.python.pointsto.PointsTo +private import semmle.python.pointsto.PointsTo2 /* Note about matching parent and child nodes and CFG splitting: @@ -212,9 +212,19 @@ class ControlFlowNode extends @py_flow_node { py_scope_flow(this, _, -1) } - /** Use ControlFlowNode.refersTo() instead. */ - deprecated Object pointsTo() { - this.refersTo(result) + /** The value that this ControlFlowNode points-to. */ + predicate pointsTo(Value value) { + this.pointsTo(_, value, _) + } + + /** The value and origin that this ControlFlowNode points-to. */ + predicate pointsTo(Value value, ControlFlowNode origin) { + this.pointsTo(_, value, origin) + } + + /** The value and origin that this ControlFlowNode points-to, given the context. */ + predicate pointsTo(Context context, Value value, ControlFlowNode origin) { + PointsTo2::points_to(this, context, value, origin) } /** Gets what this flow node might "refer-to". Performs a combination of localized (intra-procedural) points-to @@ -222,37 +232,40 @@ class ControlFlowNode extends @py_flow_node { * precise, but may not provide information for a significant number of flow-nodes. * If the class is unimportant then use `refersTo(value)` or `refersTo(value, origin)` instead. */ - predicate refersTo(Object value, ClassObject cls, ControlFlowNode origin) { - not py_special_objects(cls, "_semmle_unknown_type") - and - not value = unknownValue() - and - PointsTo::points_to(this, _, value, cls, origin) + predicate refersTo(Object obj, ClassObject cls, ControlFlowNode origin) { + this.refersTo(_, obj, cls, origin) } /** Gets what this expression might "refer-to" in the given `context`. */ - predicate refersTo(Context context, Object value, ClassObject cls, ControlFlowNode origin) { - not py_special_objects(cls, "_semmle_unknown_type") - and - PointsTo::points_to(this, context, value, cls, origin) + predicate refersTo(Context context, Object obj, ClassObject cls, ControlFlowNode origin) { + exists(Value value | + PointsTo2::points_to(this, context, value, origin) and + cls = value.getClass().getSource() | + if exists(value.getSource().(Object)) then + obj = value.getSource() + else + obj = origin + ) } /** Whether this flow node might "refer-to" to `value` which is from `origin` * Unlike `this.refersTo(value, _, origin)` this predicate includes results * where the class cannot be inferred. */ - predicate refersTo(Object value, ControlFlowNode origin) { - PointsTo::points_to(this, _, value, _, origin) - and - not value = unknownValue() + predicate refersTo(Object obj, ControlFlowNode origin) { + exists(Value value | + PointsTo2::points_to(this, _, value, origin) | + if exists(value.getSource().(Object)) then + obj = value.getSource() + else + obj = origin + ) } /** Equivalent to `this.refersTo(value, _)` */ - predicate refersTo(Object value) { - PointsTo::points_to(this, _, value, _, _) - and - not value = unknownValue() + predicate refersTo(Object obj) { + this.refersTo(obj, _) } /** Gets the basic block containing this flow node */ diff --git a/python/ql/src/semmle/python/Import.qll b/python/ql/src/semmle/python/Import.qll index 58c42fa0e296..b0f3d3a55334 100644 --- a/python/ql/src/semmle/python/Import.qll +++ b/python/ql/src/semmle/python/Import.qll @@ -14,7 +14,7 @@ class Alias extends Alias_ { private predicate valid_module_name(string name) { exists(Module m | m.getName() = name) or - exists(Builtin cmod | cmod.getClass() = theModuleType().asBuiltin() and cmod.getName() = name) + exists(Builtin cmod | cmod.getClass() = Builtin::special("ModuleType") and cmod.getName() = name) } /** An artificial expression representing an import */ diff --git a/python/ql/src/semmle/python/Module.qll b/python/ql/src/semmle/python/Module.qll index ec62088cc94e..ae9fa6c87e5c 100644 --- a/python/ql/src/semmle/python/Module.qll +++ b/python/ql/src/semmle/python/Module.qll @@ -1,5 +1,6 @@ import python -private import semmle.python.pointsto.PointsTo +private import semmle.python.objects.ObjectAPI +private import semmle.python.objects.Modules /** A module. This is the top level element in an AST, corresponding to a source file. * It is also a Scope; the scope of global variables. */ @@ -66,7 +67,10 @@ class Module extends Module_, Scope, AstNode { string getAnExport() { py_exports(this, result) or - not PointsTo::module_defines_name(this, "__all__") and PointsTo::module_defines_name(this, result) + exists(ModuleValue mod | + mod.getSource() = this.getEntryNode() | + not mod.exports(result) + ) } /** Gets the source file for this module */ diff --git a/python/ql/src/semmle/python/SelfAttribute.qll b/python/ql/src/semmle/python/SelfAttribute.qll index b27f866170cb..f0b8b8cbdfb8 100644 --- a/python/ql/src/semmle/python/SelfAttribute.qll +++ b/python/ql/src/semmle/python/SelfAttribute.qll @@ -3,7 +3,6 @@ */ import python -private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.Filters /** An attribute access where the left hand side of the attribute expression diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 2a12f0a20cda..fcbedfba8185 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -4,7 +4,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo2 -private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.pointsto.PointsToContext private import semmle.python.pointsto.MRO2 private import semmle.python.types.Builtins @@ -54,7 +54,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti result = this.getScope().toString() } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { this = TPythonFunctionObject(node) and context.appliesTo(node) } @@ -72,7 +72,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti this = TPythonFunctionObject(result) } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { exists(Function func, ControlFlowNode rval | func = this.getScope() and callee.appliesToScope(func) and @@ -105,7 +105,7 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc result = "Builtin-function " + this.getBuiltin().getName() } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { none() } @@ -115,7 +115,7 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc override boolean isComparable() { result = true } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -185,13 +185,13 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod result = TBuiltinClassObject(this.getBuiltin().getClass()) } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { none() } override boolean isComparable() { result = true } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -236,13 +236,13 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { result = TBuiltinClassObject(Builtin::special("MethodType")) } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { this = TBoundMethod(node, _, _, context) } override boolean isComparable() { result = false } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { this.getFunction().callResult(callee, obj, origin) } @@ -264,6 +264,103 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { } +class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { + + override string toString() { + result = "classmethod()" + } + + override boolean booleanValue() { result = true } + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + exists(CallableObjectInternal function | + this = TClassMethod(node, function) and + class_method(node, function, context) + ) + } + + CallableObjectInternal getFunction() { + this = TClassMethod(_, result) + } + + override ClassDecl getClassDeclaration() { none() } + + override boolean isClass() { result = false } + + override ObjectInternal getClass() { result = ObjectInternal::builtin("classmethod") } + + override boolean isComparable() { none() } + + override Builtin getBuiltin() { none() } + + override ControlFlowNode getOrigin() { this = TClassMethod(result, _) } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } + + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } + + override int intValue() { none() } + + override string strValue() { none() } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + this.getFunction().calleeAndOffset(scope, paramOffset) + } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate attributesUnknown() { none() } + +} + +class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { + + override string toString() { + result = "staticmethod()" + } + + override boolean booleanValue() { result = true } + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + exists(CallableObjectInternal function | + this = TStaticMethod(node, function) and + static_method(node, function, context) + ) + } + + CallableObjectInternal getFunction() { + this = TStaticMethod(_, result) + } + + override ClassDecl getClassDeclaration() { none() } + + override boolean isClass() { result = false } + + override ObjectInternal getClass() { result = ObjectInternal::builtin("staticmethod") } + + override boolean isComparable() { none() } + + override Builtin getBuiltin() { none() } + + override ControlFlowNode getOrigin() { this = TStaticMethod(result, _) } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } + + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } + + override int intValue() { none() } + + override string strValue() { none() } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + this.getFunction().calleeAndOffset(scope, paramOffset) + } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate attributesUnknown() { none() } + +} diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 6aa308ea9a17..cf1af47ffb2b 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -4,7 +4,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo2 -private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.pointsto.PointsToContext private import semmle.python.pointsto.MRO2 private import semmle.python.types.Builtins @@ -48,7 +48,7 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject result = "class " + this.getScope().getName() } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { this = TPythonClassObject(node) and context.appliesTo(node) } @@ -85,7 +85,7 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject override predicate attributesUnknown() { none() } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -108,7 +108,7 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec result = "builtin-class " + this.getBuiltin().getName() } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { none() } @@ -135,7 +135,7 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec override predicate attributesUnknown() { none() } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -160,10 +160,10 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { } override ObjectInternal getClass() { - result = TUnknownClass() + result = this } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { none() } @@ -173,7 +173,7 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { none() } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index c8deab2657eb..d5aa4a790c01 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -3,7 +3,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo2 -private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins @@ -30,7 +30,7 @@ abstract class BooleanObjectInternal extends ObjectInternal { none() } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { // Booleans aren't callable none() } @@ -66,7 +66,7 @@ class TrueObjectInternal extends BooleanObjectInternal, TTrue { result = true } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { node.(NameNode).getId() = "True" and context.appliesTo(node) } @@ -78,6 +78,10 @@ class TrueObjectInternal extends BooleanObjectInternal, TTrue { none() } + override @py_object getSource() { + result = Builtin::special("True") + } + } class FalseObjectInternal extends BooleanObjectInternal, TFalse { @@ -90,7 +94,7 @@ class FalseObjectInternal extends BooleanObjectInternal, TFalse { result = false } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { node.(NameNode).getId() = "False" and context.appliesTo(node) } @@ -102,6 +106,10 @@ class FalseObjectInternal extends BooleanObjectInternal, TFalse { none() } + override @py_object getSource() { + result = Builtin::special("False") + } + } class NoneObjectInternal extends ObjectInternal, TNone { @@ -127,7 +135,7 @@ class NoneObjectInternal extends ObjectInternal, TNone { result = TBuiltinClassObject(Builtin::special("NoneType")) } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { node.(NameNode).getId() = "None" and context.appliesTo(node) } @@ -135,16 +143,16 @@ class NoneObjectInternal extends ObjectInternal, TNone { none() } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { // None isn't callable none() } - + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { // None isn't callable none() } - + override ControlFlowNode getOrigin() { none() } @@ -167,6 +175,10 @@ class NoneObjectInternal extends ObjectInternal, TNone { override predicate attributesUnknown() { none() } + override @py_object getSource() { + result = Builtin::special("None") + } + } @@ -176,7 +188,7 @@ class IntObjectInternal extends ObjectInternal, TInt { result = "int " + this.intValue().toString() } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { context.appliesTo(node) and node.getNode().(IntegerLiteral).getValue() = this.intValue() } @@ -198,7 +210,7 @@ class IntObjectInternal extends ObjectInternal, TInt { none() } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { // ints aren't callable none() } @@ -236,6 +248,10 @@ class IntObjectInternal extends ObjectInternal, TInt { override predicate attributesUnknown() { none() } + override @py_object getSource() { + result.(Builtin).intValue() = this.intValue() + } + } @@ -245,7 +261,7 @@ class StringObjectInternal extends ObjectInternal, TString { result = "'" + this.strValue() + "'" } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { context.appliesTo(node) and node.getNode().(StrConst).getText() = this.strValue() } @@ -266,7 +282,7 @@ class StringObjectInternal extends ObjectInternal, TString { none() } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { // strings aren't callable none() } @@ -304,6 +320,10 @@ class StringObjectInternal extends ObjectInternal, TString { override predicate attributesUnknown() { none() } + override @py_object getSource() { + result.(Builtin).strValue() = this.strValue() + } + } diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 596769d4dcc0..71916c18a051 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -4,7 +4,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo2 -private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { @@ -19,7 +19,7 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { result = maybe() } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { this = TSpecificInstance(node, _, context) } @@ -53,7 +53,7 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { this = TSpecificInstance(result, _, _) } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -96,7 +96,7 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { result = maybe() } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { context.appliesTo(node) and this.getClass() = ObjectInternal::builtin("float") and node.getNode() instanceof FloatLiteral @@ -132,7 +132,7 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { none() } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -171,7 +171,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal { override boolean booleanValue() { result = true } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { exists(ObjectInternal self, ClassObjectInternal startclass | super_instantiation(node, self, startclass, context) and this = TSuperInstance(self, startclass) @@ -190,7 +190,9 @@ class SuperInstance extends TSuperInstance, ObjectInternal { override boolean isClass() { result = false } - override ObjectInternal getClass() { none() } + override ObjectInternal getClass() { + result = ObjectInternal::builtin("super") + } override boolean isComparable() { none() } @@ -202,7 +204,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal { override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { none() } + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } override int intValue() { none() } diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 920f9e45b6e5..862a98e64321 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -3,7 +3,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo2 -private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins abstract class ModuleObjectInternal extends ObjectInternal { @@ -12,7 +12,7 @@ abstract class ModuleObjectInternal extends ObjectInternal { abstract Module getSourceModule(); - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { // Modules aren't callable none() } @@ -34,6 +34,10 @@ abstract class ModuleObjectInternal extends ObjectInternal { result = true } + override ObjectInternal getClass() { + result = ObjectInternal::moduleType() + } + } class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleObject { @@ -50,7 +54,7 @@ class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleOb result = this.getBuiltin().getName() } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { none() } @@ -58,10 +62,6 @@ class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleOb none() } - override ObjectInternal getClass() { - result = TBuiltinClassObject(this.getBuiltin().getClass()) - } - override Module getSourceModule() { none() } @@ -93,19 +93,19 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { none() } - Folder getFolder() { - this = TPackageObject(result) - } - override string toString() { result = "Package " + this.getName() } + Folder getFolder() { + this = TPackageObject(result) + } + override string getName() { result = moduleNameFromFile(this.getFolder()) } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { none() } @@ -113,10 +113,6 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { none() } - override ObjectInternal getClass() { - result = TBuiltinClassObject(this.getBuiltin().getClass()) - } - override Module getSourceModule() { result.getFile() = this.getFolder().getFile("__init__.py") } @@ -191,14 +187,14 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { } override string toString() { - result = "Module " + this.getName() + result = this.getSourceModule().toString() } override string getName() { result = this.getSourceModule().getName() } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { none() } @@ -206,10 +202,6 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { none() } - override ObjectInternal getClass() { - result = TBuiltinClassObject(this.getBuiltin().getClass()) - } - override Module getSourceModule() { this = TPythonModule(result) } @@ -231,7 +223,7 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { } override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - exists(EssaVariable var, ControlFlowNode exit, PointsToContext2 imp | + exists(EssaVariable var, ControlFlowNode exit, PointsToContext imp | exit = this.getSourceModule().getANormalExit() and var.getAUse() = exit and var.getSourceVariable().getName() = name and PointsTo2::ssa_variable_points_to(var, imp, value, origin) and @@ -241,7 +233,7 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { // TO DO, dollar variable... //or //not exists(EssaVariable var | var.getAUse() = m.getANormalExit() and var.getSourceVariable().getName() = name) and - //exists(EssaVariable var, PointsToContext2 imp | + //exists(EssaVariable var, PointsToContext imp | // var.getAUse() = m.getANormalExit() and isModuleStateVariable(var) | // PointsTo2::ssa_variable_named_attribute_points_to(var, imp, name, obj, origin) and // imp.isImport() and obj != ObjectInternal::undefined() diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 11850164117b..207e5884b1ad 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -2,20 +2,28 @@ import python private import TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo2 -private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.pointsto.PointsToContext + +class ObjectSource = Object; class Value extends TObject { + Value() { + not this = ObjectInternal::unknown() and + not this = ObjectInternal::unknownClass() and + not this = ObjectInternal::undefined() + } + string toString() { result = this.(ObjectInternal).toString() } - ControlFlowNode getAReferent() { + ControlFlowNode getAReference() { PointsTo2::points_to(result, _, this, _) } - predicate pointsTo(ControlFlowNode referent, PointsToContext2 context, ControlFlowNode origin) { - PointsTo2::points_to(referent, context, this, origin) + predicate pointsTo(ControlFlowNode node, PointsToContext context, ControlFlowNode origin) { + PointsTo2::points_to(node, context, this, origin) } Value getClass() { @@ -35,4 +43,71 @@ class Value extends TObject { this.(ObjectInternal).attribute(name, result, _) } + /* For backwards compatibility with old API */ + ObjectSource getSource() { + result = this.(ObjectInternal).getSource() + } + + /** Gets the `ControlFlowNode` that will be passed as the nth argument to `this` when called at `call`. + This predicate will correctly handle `x.y()`, treating `x` as the zeroth argument. + */ + ControlFlowNode getArgumentForCall(CallNode call, int n) { + // TO DO.. + none() + } + + /** Gets the `ControlFlowNode` that will be passed as the named argument to `this` when called at `call`. + This predicate will correctly handle `x.y()`, treating `x` as the self argument. + */ + ControlFlowNode getNamedArgumentForCall(CallNode call, string name) { + // TO DO.. + none() + } +} + +class ModuleValue extends Value { + + ModuleValue() { + this instanceof ModuleObjectInternal + } + + predicate exports(string name) { + not this.(ModuleObjectInternal).attribute("__all__", _, _) and this.(ModuleObjectInternal).attribute(name, _, _) + } + + string getName() { + result = this.(ModuleObjectInternal).getName() + } + +} + +module Module { + + ModuleValue named(string name) { + result.getName() = name + } + +} + + +class CallableValue extends Value { + + CallableValue() { + this instanceof CallableObjectInternal + } + + predicate neverReturns() { + // TO DO.. + none() + } + } + +class ClassValue extends Value { + + ClassValue() { + this.(ObjectInternal).isClass() = true + } + +} + diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 259635d81de0..8e26d3f330c1 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -2,7 +2,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.pointsto.PointsTo2 -private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins import semmle.python.objects.Modules import semmle.python.objects.Classes @@ -18,7 +18,7 @@ class ObjectInternal extends TObject { * true and false if the "object" represents a set of possible objects. */ abstract boolean booleanValue(); - abstract predicate introduced(ControlFlowNode node, PointsToContext2 context); + abstract predicate introduced(ControlFlowNode node, PointsToContext context); /** Gets the class declaration for this object, if it is a declared class. */ abstract ClassDecl getClassDeclaration(); @@ -35,15 +35,13 @@ class ObjectInternal extends TObject { abstract boolean isComparable(); /** Gets the `Builtin` for this object, if any. - * All objects (except unknown and undefined values) should return + * Objects (except unknown and undefined values) should attempt to return * exactly one result for either this method or `getOrigin()`. */ abstract Builtin getBuiltin(); - /** Gets a control flow node that represents the source origin of this + /** Gets a control flow node that represents the source origin of this * objects. - * All objects (except unknown and undefined values) should return - * exactly one result for either this method or `getBuiltin()`. */ abstract ControlFlowNode getOrigin(); @@ -55,7 +53,7 @@ class ObjectInternal extends TObject { /** Holds if `obj` is the result of calling `this` and `origin` is * the origin of `obj` with callee context `callee`. */ - abstract predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin); + abstract predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin); /** The integer value of things that have integer values. * That is, ints and bools. @@ -77,6 +75,16 @@ class ObjectInternal extends TObject { abstract predicate attributesUnknown(); + /** For backwards compatibility shim -- Not all objects have a "source" + * Objects (except unknown and undefined values) should attempt to return + * exactly one result for either this method`. + * */ + @py_object getSource() { + result = this.getOrigin() + or + result = this.getBuiltin() + } + } @@ -87,7 +95,7 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { } override string toString() { - result = this.getBuiltin().toString() + result = this.getBuiltin().getClass().getName() + " object" } override boolean booleanValue() { @@ -105,13 +113,13 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { result = TBuiltinClassObject(this.getBuiltin().getClass()) } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { none() } override boolean isComparable() { result = false } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -165,7 +173,7 @@ class UnknownInternal extends ObjectInternal, TUnknown { result = TUnknownClass() } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { none() } @@ -179,7 +187,7 @@ class UnknownInternal extends ObjectInternal, TUnknown { obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -229,7 +237,7 @@ class UndefinedInternal extends ObjectInternal, TUndefined { none() } - override predicate introduced(ControlFlowNode node, PointsToContext2 context) { + override predicate introduced(ControlFlowNode node, PointsToContext context) { none() } @@ -237,7 +245,7 @@ class UndefinedInternal extends ObjectInternal, TUndefined { none() } - override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -334,6 +342,9 @@ module ObjectInternal { result = TBuiltinClassObject(Builtin::special("StaticMethod")) } + ObjectInternal moduleType() { + result = TBuiltinClassObject(Builtin::special("ModuleType")) + } } /** Helper for boolean predicates returning both `true` and `false` */ diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 1f2433fea901..cba68bd2e728 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -2,7 +2,7 @@ import python private import semmle.python.types.Builtins private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo2 -private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.pointsto.PointsToContext newtype TObject = TBuiltinClassObject(Builtin bltn) { @@ -31,7 +31,9 @@ newtype TObject = classexpr.getNode() instanceof ClassExpr } or - TPackageObject(Folder f) + TPackageObject(Folder f) { + exists(moduleNameFromFile(f)) + } or TPythonModule(Module m) { not m.isPackage() } or @@ -72,7 +74,7 @@ newtype TObject = s = "__main__" } or - TSpecificInstance(ControlFlowNode instantiation, ClassObjectInternal cls, PointsToContext2 context) { + TSpecificInstance(ControlFlowNode instantiation, ClassObjectInternal cls, PointsToContext context) { PointsTo2::points_to(instantiation.(CallNode).getFunction(), context, cls, _) and cls.isSpecial() = false or @@ -80,22 +82,40 @@ newtype TObject = self_parameter(instantiation.getNode(), context, cls) } or - TBoundMethod(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext2 context) { + TBoundMethod(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext context) { method_binding(instantiation, self, function, context) } or - TUnknownInstance(ClassObjectInternal cls) { cls != TUnknownClass() } + TUnknownInstance(BuiltinClassObjectInternal cls) { any() } or TSuperInstance(ObjectInternal self, ClassObjectInternal startclass) { super_instantiation(_, self, startclass, _) } + or + TClassMethod(CallNode instantiation, CallableObjectInternal function) { + class_method(instantiation, function, _) + } + or + TStaticMethod(CallNode instantiation, CallableObjectInternal function) { + static_method(instantiation, function, _) + } private predicate is_power_2(int n) { n = 1 or exists(int half | is_power_2(half) and n = half*2) } -predicate super_instantiation(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext2 context) { +predicate static_method(CallNode instantiation, CallableObjectInternal function, PointsToContext context) { + PointsTo2::points_to(instantiation.getFunction(), context, ObjectInternal::builtin("staticmethod"), _) and + PointsTo2::points_to(instantiation.getArg(0), context, function, _) +} + +predicate class_method(CallNode instantiation, CallableObjectInternal function, PointsToContext context) { + PointsTo2::points_to(instantiation.getFunction(), context, ObjectInternal::builtin("classmethod"), _) and + PointsTo2::points_to(instantiation.getArg(0), context, function, _) +} + +predicate super_instantiation(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext context) { PointsTo2::points_to(instantiation.getFunction(), context, ObjectInternal::builtin("super"), _) and ( PointsTo2::points_to(instantiation.getArg(0), context, startclass, _) and @@ -113,7 +133,7 @@ predicate super_instantiation(CallNode instantiation, ObjectInternal self, Class ) } -predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext2 context) { +predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext context) { exists(ObjectInternal obj, string name | receiver(instantiation, context, obj, name) | exists(ObjectInternal cls | @@ -134,13 +154,13 @@ predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableOb /** Helper for method_binding */ pragma [noinline] -predicate receiver(AttrNode instantiation, PointsToContext2 context, ObjectInternal obj, string name) { +predicate receiver(AttrNode instantiation, PointsToContext context, ObjectInternal obj, string name) { PointsTo2::points_to(instantiation.getObject(name), context, obj, _) } /** Helper self parameters: `def meth(self, ...): ...`. */ pragma [noinline] -private predicate self_parameter(Parameter def, PointsToContext2 context, PythonClassObjectInternal cls) { +private predicate self_parameter(Parameter def, PointsToContext context, PythonClassObjectInternal cls) { def.isSelf() and exists(Function scope | def.(Name).getScope() = scope and diff --git a/python/ql/src/semmle/python/pointsto/MRO2.qll b/python/ql/src/semmle/python/pointsto/MRO2.qll index 65b5a8ed0df5..4137c0352ab7 100644 --- a/python/ql/src/semmle/python/pointsto/MRO2.qll +++ b/python/ql/src/semmle/python/pointsto/MRO2.qll @@ -21,7 +21,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo2 -private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins @@ -99,6 +99,10 @@ class ClassList extends TClassList { result = this.getTail().getItem(n-1) } + ClassObjectInternal getAnItem() { + result = this.getItem(_) + } + pragma [inline] ClassList removeHead(ClassObjectInternal cls) { this.getHead() = cls and result = this.getTail() diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll deleted file mode 100644 index 5f8e533a0f57..000000000000 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ /dev/null @@ -1,3032 +0,0 @@ -/** - * Part of the combined points-to, call-graph and type-inference library. - * The main relation `points_to(node, context, object, cls, origin)` relates a control flow node - * to the possible objects it points-to the inferred types of those objects and the 'origin' - * of those objects. The 'origin' is the point in source code that the object can be traced - * back to. - * - * The predicates provided are not intended to be used directly (although they are available to the advanced user), but are exposed in the user API as methods on key classes. - * - * For example, the most important predicate in the points-to relation is: - * ```ql - * predicate PointsTo::points_to(ControlFlowNode f, PointsToContext ctx, Object value, ClassObject cls, ControlFlowNode origin) - * ``` - * Where `f` is the control flow node representing something that might hold a value. `ctx` is the context in which `f` "points-to" `value` and may be "general" or from a specific call-site. - * `value` is a static approximation to a value, such as a number, a class, or an object instantiation. - * `cls` is the class of this value if known, or `theUnknownType()` which is an internal `ClassObject` and should not be exposed to the general QL user. - * `origin` is the point in the source from where `value` originates and is useful when presenting query results. - * - * The `PointsTo::points_to` relation is exposed at the user API level as - * ```ql - * ControlFlowNode.refersTo(Context context, Object value, ClassObject cls, ControlFlowNode origin) - * ``` - * - */ - -import python -private import PointsToContext -private import Base -private import semmle.python.types.Extensions -private import semmle.python.types.Builtins -private import Filters as BaseFilters -import semmle.dataflow.SSA -private import MRO - -/* Use this version for speed */ -library class CfgOrigin extends @py_object { - - string toString() { - /* Not to be displayed */ - none() - } - - /** Get a `ControlFlowNode` from `this` or `here`. - * If `this` is a ControlFlowNode then use that, otherwise fall back on `here` - */ - pragma[inline] - ControlFlowNode asCfgNodeOrHere(ControlFlowNode here) { - result = this - or - not this instanceof ControlFlowNode and result = here - } - - ControlFlowNode toCfgNode() { - result = this - } - - pragma[inline] - CfgOrigin fix(ControlFlowNode here) { - if this = unknownValue() then - result = here - else - result = this - } - -} - -/* Use this version for stronger type-checking */ -//private newtype TCfgOrigin = -// TUnknownOrigin() -// or -// TCfgOrigin(ControlFlowNode f) -// -//library class CfgOrigin extends TCfgOrigin { -// -// string toString() { -// /* Not to be displayed */ -// none() -// } -// -// /** Get a `ControlFlowNode` from `this` or `here`. -// * If `this` is a ControlFlowNode then use that, otherwise fall back on `here` -// */ -// pragma[inline] -// ControlFlowNode asCfgNodeOrHere(ControlFlowNode here) { -// this = TUnknownOrigin() and result = here -// or -// this = TCfgOrigin(result) -// } -// -// ControlFlowNode toCfgNode() { -// this = TCfgOrigin(result) -// } -// -// CfgOrigin fix(ControlFlowNode here) { -// this = TUnknownOrigin() and result = TCfgOrigin(here) -// or -// not this = TUnknownOrigin() and result = this -// } -//} -// - - -module CfgOrigin { - - CfgOrigin fromCfgNode(ControlFlowNode f) { - result = f - } - - CfgOrigin unknown() { - result = unknownValue() - } - -} - -module PointsTo { - - cached module API { - - /** INTERNAL -- Use `FunctionObject.getACall()`. - * - * Gets a call to `func` with the given context. */ - cached CallNode get_a_call(FunctionObject func, PointsToContext context) { - function_call(func, context, result) - or - method_call(func, context, result) - } - - /** INTERNAL -- Use `FunctionObject.getAFunctionCall()`. - * - * Holds if `call` is a function call to `func` with the given context. */ - cached predicate function_call(FunctionObject func, PointsToContext context, CallNode call) { - points_to(call.getFunction(), context, func, _, _) - } - - /** INTERNAL -- Use `FunctionObject.getAMethodCall()`. - * - * Holds if `call` is a method call to `func` with the given context. */ - cached predicate method_call(FunctionObject func, PointsToContext context, CallNode call) { - Calls::plain_method_call(func, context, call) - or - Calls::super_method_call(context, call, _, func) - or - class_method_call(_, _, func, context, call) - } - - /** INTERNAL -- Use `ClassMethod.getACall()` instead */ - cached predicate class_method_call(Object cls_method, ControlFlowNode attr, FunctionObject func, PointsToContext context, CallNode call) { - exists(ClassObject cls, string name | - attr = call.getFunction() and - Types::class_attribute_lookup(cls, name, cls_method, _, _) | - Calls::receiver_type_for(cls, name, attr, context) - or - points_to(attr.(AttrNode).getObject(name), context, cls, _, _) - ) and - class_method(cls_method, func) - } - - /** INTERNAL -- Use `ClassMethod` instead */ - cached predicate class_method(Object cls_method, FunctionObject method) { - decorator_call(cls_method, theClassMethodType(), method) - } - - pragma [nomagic] - private predicate decorator_call(Object method, ClassObject decorator, FunctionObject func) { - exists(CallNode f, PointsToContext imp | - method = f and imp.isImport() and - points_to(f.getFunction(), imp, decorator, _, _) and - points_to(f.getArg(0), imp, func, _, _) - ) - } - - /** INTERNAL -- Use `f.refersTo(value, cls, origin)` instead. */ - cached predicate points_to(ControlFlowNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - points_to_candidate(f, context, value, cls, origin) and - Layer::reachableBlock(f.getBasicBlock(), context) - } - - /** Gets the value that `expr` evaluates to (when converted to a boolean) when `use` refers to `(val, cls, origin)` - * and `expr` is a test (a branch) and contains `use`. */ - cached boolean test_evaluates_boolean(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, Object val, ClassObject cls, ControlFlowNode origin) { - test_contains(expr, use) and - result = Filters::evaluates(expr, use, context, val, cls, origin) - } - - /** INTERNAL -- Do not use. - * - * Holds if `package.name` points to `(value, cls, origin)`, where `package` is a package object. */ - cached predicate package_attribute_points_to(PackageObject package, string name, Object value, ClassObject cls, CfgOrigin origin) { - py_module_attributes(package.getInitModule().getModule(), name, value, cls, origin) - or - exists(Module init | - init = package.getInitModule().getModule() and - not exists(EssaVariable var | var.getAUse() = init.getANormalExit() and var.getSourceVariable().getName() = name) and - exists(EssaVariable var, Context context | - isModuleStateVariable(var) and var.getAUse() = init.getANormalExit() and - context.isImport() and - SSA::ssa_variable_named_attribute_points_to(var, context, name, undefinedVariable(), _, _) and - origin = value and - value = package.submodule(name) and - cls = theModuleType() - ) - ) - or - package.hasNoInitModule() and - value = package.submodule(name) and cls = theModuleType() and origin = CfgOrigin::fromCfgNode(value) - } - - /** INTERNAL -- `Use m.attributeRefersTo(name, obj, origin)` instead. - * - * Holds if `m.name` points to `(value, cls, origin)`, where `m` is a (source) module. */ - cached predicate py_module_attributes(Module m, string name, Object obj, ClassObject cls, CfgOrigin origin) { - exists(EssaVariable var, ControlFlowNode exit, PointsToContext imp | - exit = m.getANormalExit() and var.getAUse() = exit and - var.getSourceVariable().getName() = name and - ssa_variable_points_to(var, imp, obj, cls, origin) and - imp.isImport() and - obj != undefinedVariable() - ) - or - not exists(EssaVariable var | var.getAUse() = m.getANormalExit() and var.getSourceVariable().getName() = name) and - exists(EssaVariable var, PointsToContext imp | - var.getAUse() = m.getANormalExit() and isModuleStateVariable(var) | - SSA::ssa_variable_named_attribute_points_to(var, imp, name, obj, cls, origin) and - imp.isImport() and obj != undefinedVariable() - ) - } - - /** INTERNAL -- Use `ModuleObject.hasAttribute(name)` - * - * Whether the module defines name. */ - cached predicate module_defines_name(Module mod, string name) { - module_defines_name_boolean(mod, name) = true - } - - /** INTERNAL -- Use `Version.isTrue()` instead. - * - * Holds if `cmp` points to a test on version that is `value`. - * For example, if `cmp` is `sys.version[0] < "3"` then for, Python 2, `value` would be `true`. */ - cached predicate version_const(CompareNode cmp, PointsToContext context, boolean value) { - exists(ControlFlowNode fv, ControlFlowNode fc, Object val | - comparison(cmp, fv, fc, _) and - points_to(cmp, context, val, _, _) and - value = val.booleanValue() - | - sys_version_info_slice(fv, context, _) - or - sys_version_info_index(fv, context, _, _) - or - sys_version_string_char0(fv, context, _, _) - or - points_to(fv, context, theSysHexVersionNumber(), _, _) - ) - or - value = version_tuple_compare(cmp, context).booleanValue() - } - - /** INTERNAL -- Use `FunctionObject.getArgumentForCall(call, position)` instead. */ - cached ControlFlowNode get_positional_argument_for_call(FunctionObject func, PointsToContext context, CallNode call, int position) { - result = Calls::get_argument_for_call_by_position(func, context, call, position) - or - exists(string name | - result = Calls::get_argument_for_call_by_name(func, context, call, name) and - func.getFunction().getArg(position).asName().getId() = name - ) - } - - /** INTERNAL -- Use `FunctionObject.getNamedArgumentForCall(call, name)` instead. */ - cached ControlFlowNode get_named_argument_for_call(FunctionObject func, PointsToContext context, CallNode call, string name) { - ( - result = Calls::get_argument_for_call_by_name(func, context, call, name) - or - exists(int position | - result = Calls::get_argument_for_call_by_position(func, context, call, position) and - func.getFunction().getArg(position).asName().getId() = name - ) - ) - } - - /** INTERNAL -- Use `FunctionObject.neverReturns()` instead. - * Whether function `func` never returns. Slightly conservative approximation, this predicate may be false - * for a function that can never return. */ - cached predicate function_never_returns(FunctionObject func) { - /* A Python function never returns if it has no normal exits that are not dominated by a - * call to a function which itself never returns. - */ - function_can_never_return(func) - or - exists(Function f | - f = func.getFunction() - | - forall(BasicBlock exit | - exit = f.getANormalExit().getBasicBlock() | - exists(FunctionObject callee, BasicBlock call | - get_a_call(callee, _).getBasicBlock() = call and - function_never_returns(callee) and - call.dominates(exit) - ) - ) - ) - } - - /** INTERNAL -- Use `m.importedAs(name)` instead. - * - * Holds if `import name` will import the module `m`. */ - cached predicate module_imported_as(ModuleObject m, string name) { - /* Normal imports */ - m.getName() = name - or - /* sys.modules['name'] = m */ - exists(ControlFlowNode sys_modules_flow, ControlFlowNode n, ControlFlowNode mod | - /* Use previous points-to here to avoid slowing down the recursion too much */ - exists(SubscriptNode sub, Object sys_modules | - sub.getValue() = sys_modules_flow and - points_to(sys_modules_flow, _, sys_modules, _, _) and - sys_modules.asBuiltin() = Builtin::special("sys").getMember("modules") and - sub.getIndex() = n and - n.getNode().(StrConst).getText() = name and - sub.(DefinitionNode).getValue() = mod and - points_to(mod, _, m, _, _) - ) - ) - } - - /** Holds if `call` is of the form `getattr(arg, "name")`. */ - cached predicate getattr(CallNode call, ControlFlowNode arg, string name) { - points_to(call.getFunction(), _, Object::builtin("getattr"), _, _) and - call.getArg(1).getNode().(StrConst).getText() = name and - arg = call.getArg(0) - } - - /** Holds if `f` is the instantiation of an object, `cls(...)`. */ - cached predicate instantiation(CallNode f, PointsToContext context, ClassObject cls) { - points_to(f.getFunction(), context, cls, _, _) and - cls != theTypeType() and - Types::callToClassWillReturnInstance(cls) - } - - /** Holds if `var` refers to `(value, cls, origin)` given the context `context`. */ - cached predicate ssa_variable_points_to(EssaVariable var, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - SSA::ssa_definition_points_to(var.getDefinition(), context, value, cls, origin) - } - - } - - predicate name_maybe_imported_from(ModuleObject mod, string name) { - exists(Module m, ImportStar s | - has_import_star(m, s, mod) | - exists(Variable var | name = var.getId() and var.getScope() = s.getScope()) - or - exists(ModuleObject other | - name_maybe_imported_from(other, name) and other.getModule() = m - ) - ) - or - exists(ImportMemberNode imp | - points_to(imp.getModule(name), _, mod, _, _) - ) - or - exists(PackageObject pack | - pack.getInitModule() = mod | - name_maybe_imported_from(pack, name) - ) - or - exists(mod.(PackageObject).submodule(name)) - or - exists(PackageObject package | - package.getInitModule() = mod and - exists(package.submodule(name)) - ) - or - module_exports(mod, name) - or - name = "__name__" - } - - private boolean module_defines_name_boolean(Module m, string name) { - exists(ModuleObject mod | - m = mod.getModule() | - exists(SsaVariable var | name = var.getId() and var.getAUse() = m.getANormalExit()) and result = true - or - name_maybe_imported_from(mod, name) and not any(ImportStar i).getScope() = m and result = false and - not exists(SsaVariable var | name = var.getId() and var.getAUse() = m.getANormalExit()) and - not exists(PackageObject pack | - pack.getInitModule() = mod and - exists(pack.submodule(name)) - ) - or - exists(Object obj | - obj != undefinedVariable() and - py_module_attributes(mod.getModule(), name, obj, _, _) - ) and result = true - or - exists(ImportStarNode isn, ModuleObject imported | - isn.getScope() = m and - points_to(isn.getModule(), _, imported, _, _) and - module_exports(imported, name) - ) and result = true - ) - or - name = "__name__" and result = true - } - - private boolean py_module_exports_boolean(ModuleObject mod, string name) { - exists(Module m | - m = mod.getModule() | - /* Explicitly declared in __all__ */ - m.declaredInAll(name) and result = true - or - /* No __all__ and name is defined and public */ - not m.declaredInAll(_) and name.charAt(0) != "_" and - result = module_defines_name_boolean(m, name) - or - /* May be imported from this module, but not declared in __all__ */ - name_maybe_imported_from(mod, name) and m.declaredInAll(_) and not m.declaredInAll(name) and - result = false - ) - } - - private boolean package_exports_boolean(PackageObject pack, string name) { - explicitly_imported(pack.submodule(name)) and - ( - pack.hasNoInitModule() - or - exists(ModuleObject init | - pack.getInitModule() = init | - not init.getModule().declaredInAll(_) and name.charAt(0) != "_" - ) - ) and result = true - or - result = module_exports_boolean(pack.getInitModule(), name) - } - - /** INTERNAL -- Use `m.exports(name)` instead. */ - cached predicate module_exports(ModuleObject mod, string name) { - module_exports_boolean(mod, name) = true - } - - private boolean module_exports_boolean(ModuleObject mod, string name) { - exists(mod.asBuiltin().getMember(name)) and - name.charAt(0) != "_" and result = true - or - result = package_exports_boolean(mod, name) - or - result = py_module_exports_boolean(mod, name) - } - - /** Predicates in this layer need to visible to the next layer, but not otherwise */ - private module Layer { - - /* Holds if BasicBlock `b` is reachable, given the context `context`. */ - predicate reachableBlock(BasicBlock b, PointsToContext context) { - context.appliesToScope(b.getScope()) and not exists(ConditionBlock guard | guard.controls(b, _)) - or - exists(ConditionBlock guard | - guard = b.getImmediatelyControllingBlock() and - reachableBlock(guard, context) - | - exists(Object value | - points_to(guard.getLastNode(), context, value, _, _) - | - guard.controls(b, _) and value.maybe() - or - guard.controls(b, true) and value.booleanValue() = true - or - guard.controls(b, false) and value.booleanValue() = false - ) - or - /* Assume the true edge of an assert is reachable (except for assert 0/False) */ - guard.controls(b, true) and - exists(Assert a, Expr test | - a.getTest() = test and - guard.getLastNode().getNode() = test and - not test instanceof ImmutableLiteral - ) - ) - } - - /* Holds if the edge `pred` -> `succ` is reachable, given the context `context`. - */ - predicate controlledReachableEdge(BasicBlock pred, BasicBlock succ, PointsToContext context) { - exists(ConditionBlock guard, Object value | - points_to(guard.getLastNode(), context, value, _, _) - | - guard.controlsEdge(pred, succ, _) and value.maybe() - or - guard.controlsEdge(pred, succ, true) and value.booleanValue() = true - or - guard.controlsEdge(pred, succ, false) and value.booleanValue() = false - ) - } - - /** Holds if `mod.name` points to `(value, cls, origin)`, where `mod` is a module object. */ - predicate module_attribute_points_to(ModuleObject mod, string name, Object value, ClassObject cls, CfgOrigin origin) { - py_module_attributes(mod.getModule(), name, value, cls, origin) - or - package_attribute_points_to(mod, name, value, cls, origin) - or - value.asBuiltin() = mod.asBuiltin().getMember(name) and - cls.asBuiltin() = value.asBuiltin().getClass() and origin = CfgOrigin::unknown() - } - - } - - import API - - /* Holds if `f` points to a test on the OS that is `value`. - * For example, if `f` is `sys.platform == "win32"` then, for Windows, `value` would be `true`. - */ - private predicate os_const(ControlFlowNode f, PointsToContext context, boolean value) { - exists(string os | - os_test(f, os, context) | - value = true and py_flags_versioned("sys.platform", os, major_version().toString()) - or - value = false and not py_flags_versioned("sys.platform", os, major_version().toString()) - ) - } - - /** Points-to before pruning. */ - pragma [nomagic] - private predicate points_to_candidate(ControlFlowNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - simple_points_to(f, value, cls, origin) and context.appliesToScope(f.getScope()) - or - f.isClass() and value = f and origin = f and context.appliesToScope(f.getScope()) and - cls = Types::class_get_meta_class(value) - or - exists(boolean b | - os_const(f, context, b) - | - value = theTrueObject() and b = true - or - value = theFalseObject() and b = false - ) and cls = theBoolType() and origin = f - or - import_points_to(f, value, origin) and cls = theModuleType() and context.appliesToScope(f.getScope()) - or - attribute_load_points_to(f, context, value, cls, origin) - or - exists(CfgOrigin orig | - getattr_points_to(f, context, value, cls, orig) and - origin = orig.asCfgNodeOrHere(f) - ) - or - if_exp_points_to(f, context, value, cls, origin) - or - from_import_points_to(f, context, value, cls, origin) - or - use_points_to(f, context, value, cls, origin) - or - def_points_to(f, context, value, cls, origin) - or - Calls::call_points_to(f, context, value, cls, origin) - or - subscript_points_to(f, context, value, cls, origin) - or - sys_version_info_slice(f, context, cls) and value = theSysVersionInfoTuple() and origin = f - or - sys_version_info_index(f, context, value, cls) and origin = f - or - sys_version_string_char0(f, context, value, cls) and origin = f - or - six_metaclass_points_to(f, context, value, cls, origin) - or - binary_expr_points_to(f, context, value, cls, origin) - or - compare_expr_points_to(f, context, value, cls, origin) - or - not_points_to(f, context, value, cls, origin) - or - value.(SuperCall).instantiation(context, f) and f = origin and cls = theSuperType() - or - value.(SuperBoundMethod).instantiation(context, f) and f = origin and cls = theBoundMethodType() - or - exists(boolean b | - b = Filters::evaluates_boolean(f, _, context, _, _, _) - | - value = theTrueObject() and b = true - or - value = theFalseObject() and b = false - ) and cls = theBoolType() and origin = f - or - f.(CustomPointsToFact).pointsTo(context, value, cls, origin) - } - - /** The ESSA variable with fast-local lookup (LOAD_FAST bytecode). */ - private EssaVariable fast_local_variable(NameNode n) { - n.isLoad() and - result.getASourceUse() = n and - result.getSourceVariable() instanceof FastLocalVariable - } - - /** The ESSA variable with name-local lookup (LOAD_NAME bytecode). */ - private EssaVariable name_local_variable(NameNode n) { - n.isLoad() and - result.getASourceUse() = n and - result.getSourceVariable() instanceof NameLocalVariable - } - - /** The ESSA variable for the global variable lookup. */ - private EssaVariable global_variable(NameNode n) { - n.isLoad() and - result.getASourceUse() = n and - result.getSourceVariable() instanceof GlobalVariable - } - - private predicate use_points_to_maybe_origin(NameNode f, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin_or_obj) { - ssa_variable_points_to(fast_local_variable(f), context, value, cls, origin_or_obj) - or - name_lookup_points_to_maybe_origin(f, context, value, cls, origin_or_obj) - or - not exists(fast_local_variable(f)) and not exists(name_local_variable(f)) and - global_lookup_points_to_maybe_origin(f, context, value, cls, origin_or_obj) - } - - pragma [noinline] - private predicate local_variable_undefined(NameNode f, PointsToContext context) { - ssa_variable_points_to(name_local_variable(f), context, undefinedVariable(), _, _) - } - - private predicate name_lookup_points_to_maybe_origin(NameNode f, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin_or_obj) { - exists(EssaVariable var | var = name_local_variable(f) | - ssa_variable_points_to(var, context, value, cls, origin_or_obj) - ) - or - local_variable_undefined(f, context) and - global_lookup_points_to_maybe_origin(f, context, value, cls, origin_or_obj) - } - - private predicate global_lookup_points_to_maybe_origin(NameNode f, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin_or_obj) { - ssa_variable_points_to(global_variable(f), context, value, cls, origin_or_obj) - or - exists(ControlFlowNode origin | - origin_or_obj = CfgOrigin::fromCfgNode(origin) - | - ssa_variable_points_to(global_variable(f), context, undefinedVariable(), _, _) and - potential_builtin_points_to(f, value, cls, origin) - or - not exists(global_variable(f)) and context.appliesToScope(f.getScope()) and - potential_builtin_points_to(f, value, cls, origin) - ) - } - - /** Gets an object pointed to by a use (of a variable). */ - private predicate use_points_to(NameNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(CfgOrigin origin_or_obj | - value != undefinedVariable() and - use_points_to_maybe_origin(f, context, value, cls, origin_or_obj) | - origin = origin_or_obj.asCfgNodeOrHere(f) - ) - } - - /** Gets an object pointed to by the definition of an ESSA variable. */ - private predicate def_points_to(DefinitionNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - points_to(f.getValue(), context, value, cls, origin) - } - - /** Holds if `f` points to `@six.add_metaclass(cls)\nclass ...`. */ - private predicate six_metaclass_points_to(ControlFlowNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(ControlFlowNode meta | - Types::six_add_metaclass(f, value, meta) and - points_to(meta, context, cls, _, _) - ) and - origin = value - } - - /** Holds if `obj.name` points to `(value, cls, orig)`. */ - pragma [noinline] - private predicate class_or_module_attribute(Object obj, string name, Object value, ClassObject cls, CfgOrigin orig) { - /* Normal class attributes */ - Types::class_attribute_lookup(obj, name, value, cls, orig) and cls != theStaticMethodType() and cls != theClassMethodType() - or - /* Static methods of the class */ - exists(CallNode sm | - Types::class_attribute_lookup(obj, name, sm, theStaticMethodType(), _) and sm.getArg(0) = value and - cls = thePyFunctionType() and orig = CfgOrigin::fromCfgNode(sm.getArg(0)) - ) - or - /* Module attributes */ - Layer::module_attribute_points_to(obj, name, value, cls, orig) - } - - /** Holds if `f` points to `(value, cls, origin)` where `f` is an instance attribute, `x.attr`. */ - pragma [nomagic] - private predicate instance_attribute_load_points_to(AttrNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - f.isLoad() and - exists(string name | - exists(CfgOrigin orig | - origin = orig.asCfgNodeOrHere(f) and - named_attribute_points_to(f.getObject(name), context, name, value, cls, orig) - ) - or - /* Static methods on the class of the instance */ - exists(CallNode sm, ClassObject icls | - points_to(f.getObject(name), context, _, icls, _) and - Types::class_attribute_lookup(icls, name, sm, theStaticMethodType(), _) and sm.getArg(0) = value and cls = thePyFunctionType() and origin = value - ) - or - /* Unknown instance attributes */ - exists(Object x, ClassObject icls, ControlFlowNode obj_node | - obj_node = f.getObject(name) and - not obj_node.(NameNode).isSelf() and - points_to(obj_node, context, x, icls, _) and - (not x instanceof ModuleObject and not x instanceof ClassObject) and - not icls.isBuiltin() and - value = unknownValue() and cls = theUnknownType() and origin = f - ) - ) - } - - pragma[noinline] - private predicate receiver_object(AttrNode f, PointsToContext context, Object cls_or_mod, string name) { - f.isLoad() and - exists(ControlFlowNode fval| - fval = f.getObject(name) and - points_to(fval, context, cls_or_mod, _, _) | - cls_or_mod instanceof ClassObject or - cls_or_mod instanceof ModuleObject - ) - } - - /** Holds if `f` is an attribute `x.attr` and points to `(value, cls, origin)`. */ - private predicate attribute_load_points_to(AttrNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - instance_attribute_load_points_to(f, context, value, cls, origin) - or - exists(Object cls_or_mod, string name, CfgOrigin orig | - receiver_object(f, context, cls_or_mod, name) and - class_or_module_attribute(cls_or_mod, name, value, cls, orig) and - origin = orig.asCfgNodeOrHere(f) - ) - or - points_to(f.getObject(), context, unknownValue(), theUnknownType(), origin) and value = unknownValue() and cls = theUnknownType() - or - exists(CustomPointsToAttribute object, string name | - points_to(f.getObject(name), context, object, _, _) and - object.attributePointsTo(name, value, cls, origin) - ) - } - - /** Holds if `f` is an expression node `tval if cond else fval` and points to `(value, cls, origin)`. */ - private predicate if_exp_points_to(IfExprNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - points_to(f.getAnOperand(), context, value, cls, origin) - } - - /** Holds if `f` is an import expression, `import mod` and points to `(value, cls, origin)`. */ - private predicate import_points_to(ControlFlowNode f, ModuleObject value, ControlFlowNode origin) { - exists(string name, ImportExpr i | - i.getAFlowNode() = f and i.getImportedModuleName() = name and - module_imported_as(value, name) and - origin = f - ) - } - - /** Holds if `f` is a "from import" expression, `from mod import x` and points to `(value, cls, origin)`. */ - pragma [nomagic] - private predicate from_import_points_to(ImportMemberNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(string name, ModuleObject mod, CfgOrigin orig | - points_to(f.getModule(name), context, mod, _, _) and - origin = orig.asCfgNodeOrHere(f) - | - mod.getSourceModule() = f.getEnclosingModule() and - not exists(EssaVariable var | var.getSourceVariable().getName() = name and var.getAUse() = f) and - exists(EssaVariable dollar | - isModuleStateVariable(dollar) and dollar.getAUse() = f and - SSA::ssa_variable_named_attribute_points_to(dollar, context, name, value, cls, orig) - ) - or - not mod.getSourceModule() = f.getEnclosingModule() and - Layer::module_attribute_points_to(mod, name, value, cls, orig) - ) - or - exists(EssaVariable var, CfgOrigin orig | - var = ssa_variable_for_module_attribute(f, context) and - ssa_variable_points_to(var, context, value, cls, orig) and - origin = orig.asCfgNodeOrHere(f) - ) - } - - private EssaVariable ssa_variable_for_module_attribute(ImportMemberNode f, PointsToContext context) { - exists(string name, ModuleObject mod, Module m | - mod.getSourceModule() = m and m = f.getEnclosingModule() and m = result.getScope() and - points_to(f.getModule(name), context, mod, _, _) and - result.getSourceVariable().getName() = name and result.getAUse() = f - ) - } - - /** Holds if `f` is of the form `getattr(x, "name")` and x.name points to `(value, cls, origin)`. */ - private predicate getattr_points_to(CallNode f, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - exists(ControlFlowNode arg, string name | - named_attribute_points_to(arg, context, name, value, cls, origin) and - getattr(f, arg, name) - ) - } - - /** Whether the module is explicitly imported somewhere. */ - private predicate explicitly_imported(ModuleObject mod) { - exists(ImportExpr ie | module_imported_as(mod, ie.getAnImportedModuleName())) - or - exists(ImportMember im | module_imported_as(mod, im.getImportedModuleName())) - } - - /** Holds if an import star exists in the module m that imports the module `imported_module`, such that the flow from the import reaches the module exit. */ - private predicate has_import_star(Module m, ImportStar im, ModuleObject imported_module) { - exists(string name | - module_imported_as(imported_module, name) and - name = im.getImportedModuleName() and - im.getScope() = m and - im.getAFlowNode().getBasicBlock().reachesExit() - ) - } - - /** Track bitwise expressions so we can handle integer flags and enums. - * Tracking too many binary expressions is likely to kill performance. - */ - private predicate binary_expr_points_to(BinaryExprNode b, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - cls = theIntType() and - exists(ControlFlowNode left, ControlFlowNode right | - bitwise_expression_node(b, left, right) and - points_to(left, context, _, cls, _) and - points_to(right, context, _, cls, _) - ) and - value = origin and origin = b - } - - pragma [noinline] - private predicate incomparable_values(CompareNode cmp, PointsToContext context) { - exists(ControlFlowNode left, ControlFlowNode right | - cmp.operands(left, _, right) and - exists(Object lobj, Object robj | - points_to(left, context, lobj, _, _) and - points_to(right, context, robj, _, _) | - not Filters::comparable_value(lobj) - or - not Filters::comparable_value(robj) - ) - ) - } - - pragma [noinline] - private Object in_tuple(CompareNode cmp, PointsToContext context) { - exists(ControlFlowNode left, ControlFlowNode right | - cmp.operands(left, any(In i), right) and - exists(Object lobj, TupleObject tuple | - points_to(left, context, lobj, _, _) and - points_to(right, context, tuple, _, _) - | - lobj = tuple.getBuiltinElement(_) and result = theTrueObject() - or - not lobj = tuple.getBuiltinElement(_) and result = theFalseObject() - ) - ) - } - - pragma [noinline] - private predicate const_compare(CompareNode cmp, PointsToContext context, int comp, boolean strict) { - exists(ControlFlowNode left, ControlFlowNode right | - inequality(cmp, left, right, strict) and - ( - exists(NumericObject n1, NumericObject n2 | - points_to(left, context, n1, _, _) and - points_to(right, context, n2, _, _) and - comp = int_compare(n1, n2) - ) - or - exists(StringObject s1, StringObject s2| - points_to(left, context, s1, _, _) and - points_to(right, context, s2, _, _) and - comp = string_compare(s1, s2) - ) - ) - ) - } - - pragma [noinline] - private Object version_tuple_compare(CompareNode cmp, PointsToContext context) { - exists(ControlFlowNode lesser, ControlFlowNode greater, boolean strict | - inequality(cmp, lesser, greater, strict) and - exists(TupleObject tuple, int comp | - points_to(lesser, context, tuple, _, _) and - points_to(greater, context, theSysVersionInfoTuple(), _, _) and - comp = version_tuple_compare(tuple) - or - points_to(lesser, context, theSysVersionInfoTuple(), _, _) and - points_to(greater, context, tuple, _, _) and - comp = version_tuple_compare(tuple)*-1 - | - comp = -1 and result = theTrueObject() - or - comp = 0 and strict = false and result = theTrueObject() - or - comp = 0 and strict = true and result = theFalseObject() - or - comp = 1 and result = theFalseObject() - ) - ) - } - - /** Holds if `cls` is an element of the tuple referred to by `f`. - * Helper for relevant_subclass_relation - */ - private predicate element_of_points_to_tuple(ControlFlowNode f, PointsToContext context, ClassObject cls) { - exists(TupleObject t | - points_to(f, context, t, _, _) | - cls = t.getBuiltinElement(_) - or - points_to(t.getSourceElement(_), _, cls, _, _) - ) - } - - private predicate compare_expr_points_to(CompareNode cmp, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - equality_expr_points_to(cmp, context, value, cls, origin) - or - cls = theBoolType() and origin = cmp and - ( - incomparable_values(cmp, context) and - (value = theFalseObject() or value = theTrueObject()) - or - value = in_tuple(cmp, context) - or - exists(int comp, boolean strict | - const_compare(cmp, context, comp, strict) - | - comp = -1 and value = theTrueObject() - or - comp = 0 and strict = false and value = theTrueObject() - or - comp = 0 and strict = true and value = theFalseObject() - or - comp = 1 and value = theFalseObject() - ) - or - value = version_tuple_compare(cmp, context) - ) - } - - pragma[inline] - private int int_compare(NumericObject n1, NumericObject n2) { - exists(int i1, int i2 | - i1 = n1.intValue() and i2 = n2.intValue() | - i1 = i2 and result = 0 - or - i1 < i2 and result = -1 - or - i1 > i2 and result = 1 - ) - } - - pragma[inline] - private int string_compare(StringObject s1, StringObject s2) { - exists(string a, string b | - a = s1.getText() and b = s2.getText() | - a = b and result = 0 - or - a < b and result = -1 - or - a > b and result = 1 - ) - } - - private predicate not_points_to(UnaryExprNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - f.getNode().getOp() instanceof Not and - cls = theBoolType() and origin = f and - exists(Object operand | - points_to(f.getOperand(), context, operand, _, _) - | - operand.maybe() and value = theTrueObject() - or - operand.maybe() and value = theFalseObject() - or - operand.booleanValue() = false and value = theTrueObject() - or - operand.booleanValue() = true and value = theFalseObject() - ) - } - - private predicate equality_expr_points_to(CompareNode cmp, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - cls = theBoolType() and origin = cmp and - exists(ControlFlowNode x, ControlFlowNode y, Object xobj, Object yobj, boolean is | - BaseFilters::equality_test(cmp, x, is, y) and - points_to(x, context, xobj, _, _) and - points_to(y, context, yobj, _, _) and - Filters::equatable_value(xobj) and Filters::equatable_value(yobj) - | - xobj = yobj and is = true and value = theTrueObject() - or - xobj != yobj and is = true and value = theFalseObject() - or - xobj = yobj and is = false and value = theFalseObject() - or - xobj != yobj and is = false and value = theTrueObject() - ) - } - - private predicate subscript_points_to(SubscriptNode sub, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(Object unknownCollection | - varargs_points_to(unknownCollection, _) or - kwargs_points_to(unknownCollection, _) - | - sub.isLoad() and - points_to(sub.getValue(), context, unknownCollection, _, _) and - value = unknownValue() and cls = theUnknownType() and origin = sub - ) - or - points_to(sub.getValue(), context, unknownValue(), _, _) and - value = unknownValue() and cls = theUnknownType() and origin = sub - } - - /* ************** - * VERSION INFO * - ****************/ - - /** Holds if `s` points to `sys.version_info[0]`. */ - private predicate sys_version_info_index(SubscriptNode s, PointsToContext context, NumericObject value, ClassObject cls) { - points_to(s.getValue(), context, theSysVersionInfoTuple(), _, _) and - exists(NumericObject zero | - zero.intValue() = 0 | - points_to(s.getIndex(), context, zero, _, _) - ) and - value.intValue() = major_version() and - cls = theIntType() - } - - /** Holds if `s` points to `sys.version_info[:x]` or `sys.version_info[:]`. */ - private predicate sys_version_info_slice(SubscriptNode s, PointsToContext context, ClassObject cls) { - points_to(s.getValue(), context, theSysVersionInfoTuple(), cls, _) and - exists(Slice index | index = s.getIndex().getNode() | - not exists(index.getStart()) - ) - } - - /** Holds if `s` points to `sys.version[0]`. */ - private predicate sys_version_string_char0(SubscriptNode s, PointsToContext context, Object value, ClassObject cls) { - points_to(s.getValue(), context, theSysVersionString(), cls, _) and - exists(NumericObject zero | - zero.intValue() = 0 | - points_to(s.getIndex(), context, zero, _, _) - ) - and - value = object_for_string(major_version().toString()) - } - - /* Version tests. Ignore micro and release parts. Treat major, minor as a single version major*10+minor - * Currently cover versions 0.9 to 4.0 - */ - - /** Helper for `version_const`. */ - private predicate comparison(CompareNode cmp, ControlFlowNode fv, ControlFlowNode fc, string opname) { - exists(Cmpop op | - cmp.operands(fv, op, fc) and opname = op.getSymbol() - or - cmp.operands(fc, op, fv) and opname = reversed(op) - ) - } - - /** Helper for `version_const`. */ - private predicate inequality(CompareNode cmp, ControlFlowNode lesser, ControlFlowNode greater, boolean strict) { - exists(Cmpop op | - cmp.operands(lesser, op, greater) and op.getSymbol() = "<" and strict = true - or - cmp.operands(lesser, op, greater) and op.getSymbol() = "<=" and strict = false - or - cmp.operands(greater, op, lesser) and op.getSymbol() = ">" and strict = true - or - cmp.operands(greater, op, lesser) and op.getSymbol() = ">=" and strict = false - ) - } - - /** Holds if `f` is a test for the O/S. */ - private predicate os_test(ControlFlowNode f, string os, PointsToContext context) { - exists(ControlFlowNode c | - os_compare(c, os) and - points_to(f, context, _, _, c) - ) - } - - predicate named_attribute_points_to(ControlFlowNode f, PointsToContext context, string name, Object value, ClassObject cls, CfgOrigin origin) { - exists(EssaVariable var | - var.getAUse() = f | - SSA::ssa_variable_named_attribute_points_to(var, context, name, value, cls, origin) - ) - or - exists(ClassObject c, EssaVariable self, Function init | - instantiation(f, context, c) and - init = c.getPyClass().getInitMethod() and - self.getAUse() = init.getANormalExit() and - SSA::ssa_variable_named_attribute_points_to(self, context, name, value, cls, origin) - ) - } - - private module Calls { - - /** Holds if `f` is a call to type() with a single argument `arg` */ - private predicate call_to_type(CallNode f, ControlFlowNode arg, PointsToContext context) { - points_to(f.getFunction(), context, theTypeType(), _, _) and not exists(f.getArg(1)) and arg = f.getArg(0) - } - - pragma [noinline] - predicate call_to_type_known_python_class_points_to(CallNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(ControlFlowNode arg | - call_to_type(f, arg, context) and - points_to(arg, context, _, value, _) - ) and - origin.getNode() = value.getOrigin() and - cls = theTypeType() - } - - pragma [noinline] - predicate call_to_type_known_builtin_class_points_to(CallNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(ControlFlowNode arg | - call_to_type(f, arg, context) | - points_to(arg, context, _, value, _) - ) and - not exists(value.getOrigin()) and - origin = f and cls = theTypeType() - } - - pragma [noinline] - predicate call_points_to_builtin_function(CallNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(BuiltinCallable b | - b != Object::builtin("isinstance") and - b != Object::builtin("issubclass") and - b != Object::builtin("callable") and - f = get_a_call(b, context) and - cls = b.getAReturnType() - ) and - f = origin and - ( - cls = theNoneType() and value = theNoneObject() - or - cls != theNoneType() and value = f - ) - } - - /** Holds if call is to an object that always returns its first argument. - * Typically, this is for known decorators and the like. - * The current implementation only accounts for instances of `zope.interface.declarations.implementer` and - * calls to `functools.wraps(fn)`. - */ - private predicate annotation_call(CallNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - points_to(f.getArg(0), context, value, cls, origin) and - ( - points_to(f.getFunction(), context, _, zopeInterfaceImplementer(), _) - or - points_to(f.getFunction().(CallNode).getFunction(), context, functoolsWraps(), _, _) - ) - } - - private ClassObject zopeInterfaceImplementer() { - result.getName() = "implementer" and - result.getPyClass().getEnclosingModule().getName() = "zope.interface.declarations" - } - - private PyFunctionObject functoolsWraps() { - result.getName() = "wraps" and - result.getFunction().getEnclosingModule().getName() = "functools" - } - - pragma [noinline] - predicate call_to_procedure_points_to(CallNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(PyFunctionObject func | - f = get_a_call(func, context) and - implicitly_returns(func, value, cls) and origin.getNode() = func.getOrigin() - ) - } - - predicate call_to_unknown(CallNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - value = unknownValue() and cls = theUnknownType() and origin = f - and - exists(ControlFlowNode callable | - callable = f.getFunction() or - callable = f.getFunction().(AttrNode).getObject() - | - points_to(callable, context, unknownValue(), _, _) - ) - } - - predicate call_to_type_new(CallNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - points_to(f.getFunction(), context, theTypeNewMethod(), _, _) and - value = theUnknownType() and cls = theUnknownType() and origin = f - } - - predicate call_to_generator_points_to(CallNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(PyFunctionObject func | - f = get_a_call(func, context) | - func.getFunction().isGenerator() and origin = f and value = f and cls = theGeneratorType() - ) - } - - /* Helper for call_points_to_python_function */ - predicate return_val_points_to(PyFunctionObject func, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(ControlFlowNode rval | - rval = func.getAReturnedNode() and - points_to(rval, context, value, cls, origin) - ) - } - - pragma [noinline] - predicate call_points_to_python_function(CallNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(PyFunctionObject func, PointsToContext callee | - return_val_points_to(func, callee, value, cls, origin) and - callee.fromCall(f, func, context) - ) - } - - /** A call, including calls to `type(arg)`, functions and classes. - * - * Call analysis logic - * =================== - * There are five possibilities (that we currently account for) here. - * 1. `type(known_type)` where we know the class of `known_type` and we know its origin - * 2. `type(known_type)` where we know the class of `known_type`, - * but we don't know its origin (because it is a builtin type) - * 3. `Class(...)` where Class is any class except type (with one argument) and calls to that class return instances of that class - * 4. `func(...)` where we know the return type of func (because it is a builtin function) - * 5. `func(...)` where we know the returned object and origin of func (because it is a Python function) - */ - predicate call_points_to(CallNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - /* Case 1 */ - call_to_type_known_python_class_points_to(f, context, value, cls, origin) - or - /* Case 2 */ - call_to_type_known_builtin_class_points_to(f, context, value, cls, origin) - or - /* Case 3 */ - instantiation(f, context, cls) and value = f and f = origin - or - /* Case 4 */ - call_points_to_builtin_function(f, context, value, cls, origin) - or - /* Case 5a */ - call_points_to_python_function(f, context, value, cls, origin) - or - /* Case 5b */ - call_to_generator_points_to(f, context, value, cls, origin) - or - /* Case 5c */ - call_to_procedure_points_to(f, context, value, cls, origin) - or - call_to_unknown(f, context, value, cls, origin) - or - call_to_type_new(f, context, value, cls, origin) - or - annotation_call(f, context, value, cls, origin) - } - - /** INTERNAL -- Public for testing only. - * Whether `call` is a call to `method` of the form `super(...).method(...)` - */ - predicate super_method_call(PointsToContext context, CallNode call, EssaVariable self, FunctionObject method) { - exists(ControlFlowNode func, SuperBoundMethod bound_method | - call.getFunction() = func and - points_to(func, context, bound_method, _, _) and - method = bound_method.getFunction(context) and - self = bound_method.getSelf() - ) - } - - /** INTERNAL -- Use `FunctionObject.getAMethodCall()`. */ - pragma [nomagic] - predicate plain_method_call(FunctionObject func, PointsToContext context, CallNode call) { - exists(ControlFlowNode attr, ClassObject cls, string name | - attr = call.getFunction() and - receiver_type_for(cls, name, attr, context) and - Types::class_attribute_lookup(cls, name, func, _, _) - ) - } - - /** INTERNAL -- Do not use; part of the internal API. - * - * Whether cls `cls` is the receiver type of an attribute access `n`. - * Also bind the name of the attribute. - */ - predicate receiver_type_for(ClassObject cls, string name, ControlFlowNode n, PointsToContext context) { - /* `super().meth()` is not a method on `super` */ - cls != theSuperType() and - exists(Object o | - /* list.__init__() is not a call to type.__init__() */ - o.notClass() | - points_to(n.(AttrNode).getObject(name), context, o, cls, _) - ) - or - exists(PlaceHolder p, Variable v | - n.getNode() = p and n.(NameNode).uses(v) and name = v.getId() and - p.getScope().getScope() = cls.getPyClass() and context.appliesTo(n) - ) - } - - /** Gets the argument for the parameter at `position` where `call` is a call to `func`. - * Handles method calls, such that for a call `x.foo()` with `position equal to 0, the result is `x`. - */ - pragma [nomagic] - ControlFlowNode get_argument_for_call_by_position(FunctionObject func, PointsToContext context, CallNode call, int position) { - method_call(func, context, call) and - ( - result = call.getArg(position-1) - or - position = 0 and result = call.getFunction().(AttrNode).getObject() - ) - or - function_call(func, context, call) and - result = call.getArg(position) - } - - /** Holds if `value` is the value attached to the keyword argument `name` in `call`. */ - predicate keyword_value_for_call(CallNode call, string name, ControlFlowNode value) { - exists(Keyword kw | - call.getNode().getAKeyword() = kw | - kw.getArg() = name and kw.getValue() = value.getNode() and - value.getBasicBlock().dominates(call.getBasicBlock()) - ) - } - - /** Gets the value for the keyword argument `name` in `call`, where `call` calls `func` in context. */ - ControlFlowNode get_argument_for_call_by_name(FunctionObject func, PointsToContext context, CallNode call, string name) { - call = get_a_call(func, context) and - keyword_value_for_call(call, name, result) - } - - /** Holds if `func` implicitly returns the `None` object */ - predicate implicitly_returns(PyFunctionObject func, Object none_, ClassObject noneType) { - noneType = theNoneType() and none_ = theNoneObject() and - exists(Function f | - f = func.getFunction() and not f.isGenerator() - | - not exists(Return ret | ret.getScope() = f) and exists(f.getANormalExit()) - or - exists(Return ret | ret.getScope() = f and not exists(ret.getValue())) - ) - } - - } - - cached module Flow { - - /** Model the transfer of values at scope-entry points. Transfer from `(pred_var, pred_context)` to `(succ_def, succ_context)`. */ - cached predicate scope_entry_value_transfer(EssaVariable pred_var, PointsToContext pred_context, ScopeEntryDefinition succ_def, PointsToContext succ_context) { - scope_entry_value_transfer_from_earlier(pred_var, pred_context, succ_def, succ_context) - or - callsite_entry_value_transfer(pred_var, pred_context, succ_def, succ_context) - or - pred_context.isImport() and pred_context = succ_context and - class_entry_value_transfer(pred_var, succ_def) - } - - /** Helper for `scope_entry_value_transfer`. Transfer of values from a temporally earlier scope to later scope. - * Earlier and later scopes are, for example, a module and functions in that module, or an __init__ method and another method. */ - pragma [noinline] - private predicate scope_entry_value_transfer_from_earlier(EssaVariable pred_var, PointsToContext pred_context, ScopeEntryDefinition succ_def, PointsToContext succ_context) { - exists(Scope pred_scope, Scope succ_scope | - BaseFlow::scope_entry_value_transfer_from_earlier(pred_var, pred_scope, succ_def, succ_scope) and - succ_context.appliesToScope(succ_scope) - | - succ_context.isRuntime() and succ_context = pred_context - or - pred_context.isImport() and pred_scope instanceof ImportTimeScope and - (succ_context.fromRuntime() or - /* A call made at import time, but from another module. Assume this module has been fully imported. */ - succ_context.isCall() and exists(CallNode call | succ_context.fromCall(call, _) and call.getEnclosingModule() != pred_scope)) - or - /* If predecessor scope is main, then we assume that any global defined exactly once - * is available to all functions. Although not strictly true, this gives less surprising - * results in practice. */ - pred_context.isMain() and pred_scope instanceof Module and succ_context.fromRuntime() and - exists(Variable v | - v = pred_var.getSourceVariable() and - not strictcount(v.getAStore()) > 1 - ) - ) - or - exists(NonEscapingGlobalVariable var | - var = pred_var.getSourceVariable() and var = succ_def.getSourceVariable() and - pred_var.getAUse() = succ_context.getRootCall() and pred_context.isImport() and - succ_context.appliesToScope(succ_def.getScope()) - ) - } - - /** Helper for `scope_entry_value_transfer`. - * Transfer of values from the callsite to the callee, for enclosing variables, but not arguments/parameters. */ - pragma [noinline] - private predicate callsite_entry_value_transfer(EssaVariable caller_var, PointsToContext caller_context, ScopeEntryDefinition entry_def, PointsToContext callee_context) { - exists(CallNode callsite, FunctionObject f, Variable var | - scope_entry_function_and_variable(entry_def, f, var) and - callee_context.fromCall(callsite, f, caller_context) and - caller_var.getSourceVariable() = var and - caller_var.getAUse() = callsite - ) - } - - /** Helper for callsite_entry_value_transfer to improve join-order */ - private predicate scope_entry_function_and_variable(ScopeEntryDefinition entry_def, FunctionObject f, Variable var) { - exists(Function func | - func = f.getFunction() | - entry_def.getDefiningNode() = func.getEntryNode() and - not var.getScope() = func and - entry_def.getSourceVariable() = var - ) - } - - /** Helper for `scope_entry_value_transfer`. */ - private predicate class_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition succ_def) { - exists(ImportTimeScope scope, ControlFlowNode class_def | - class_def = pred_var.getAUse() and - scope.entryEdge(class_def, succ_def.getDefiningNode()) and - pred_var.getSourceVariable() = succ_def.getSourceVariable() - ) - } - - /** Gets the ESSA variable from which `def` acquires its value, when a call occurs. - * Helper for `callsite_points_to`. */ - cached predicate callsite_exit_value_transfer(EssaVariable callee_var, PointsToContext callee_context, CallsiteRefinement def, PointsToContext callsite_context) { - exists(FunctionObject func, Variable var | - callee_context.fromCall(def.getCall(), func, callsite_context) and - def.getSourceVariable() = var and - var_at_exit(var, func, callee_var) - ) - } - - /* Helper for callsite_exit_value_transfer */ - private predicate var_at_exit(Variable var, FunctionObject func, EssaVariable evar) { - not var instanceof LocalVariable and - evar.getSourceVariable() = var and - evar.getScope() = func.getFunction() and - BaseFlow::reaches_exit(evar) - } - - /** Holds if the `(argument, caller)` pair matches up with `(param, callee)` pair across call. */ - cached predicate callsite_argument_transfer(ControlFlowNode argument, PointsToContext caller, ParameterDefinition param, PointsToContext callee) { - exists(CallNode call, PyFunctionObject func, int n, int offset | - callsite_calls_function(call, caller, func, callee, offset) and - argument = call.getArg(n) and - param = func.getParameter(n+offset) - ) - } - - cached predicate callsite_calls_function(CallNode call, PointsToContext caller, PyFunctionObject func, PointsToContext callee, int parameter_offset) { - /* Functions */ - callee.fromCall(call, func, caller) and - function_call(func, caller, call) and - parameter_offset = 0 - or - /* Methods */ - callee.fromCall(call, func, caller) and - method_call(func, caller, call) and - parameter_offset = 1 - or - /* Classes */ - exists(ClassObject cls | - instantiation(call, caller, cls) and - Types::class_attribute_lookup(cls, "__init__", func, _, _) and - parameter_offset = 1 and - callee.fromCall(call, caller) - ) - } - - /** Helper for `import_star_points_to`. */ - cached predicate module_and_name_for_import_star(ModuleObject mod, string name, ImportStarRefinement def, PointsToContext context) { - points_to(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _, _) and - name = def.getSourceVariable().getName() - } - - /** Holds if `def` is technically a definition of `var`, but the `from ... import *` does not in fact define `var`. */ - cached predicate variable_not_redefined_by_import_star(EssaVariable var, PointsToContext context, ImportStarRefinement def) { - var = def.getInput() and - exists(ModuleObject mod | - points_to(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _, _) | - module_exports_boolean(mod, var.getSourceVariable().getName()) = false - or - exists(Module m, string name | - m = mod.getModule() and name = var.getSourceVariable().getName() | - not m.declaredInAll(_) and name.charAt(0) = "_" - ) - ) - } - - } - - private module SSA { - - - /** Holds if the phi-function `phi` refers to `(value, cls, origin)` given the context `context`. */ - pragma [noinline] - private predicate ssa_phi_points_to(PhiFunction phi, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - exists(EssaVariable input, BasicBlock pred | - input = phi.getInput(pred) and - ssa_variable_points_to(input, context, value, cls, origin) - | - Layer::controlledReachableEdge(pred, phi.getBasicBlock(), context) - or - not exists(ConditionBlock guard | guard.controlsEdge(pred, phi.getBasicBlock(), _)) - ) - or - ssa_variable_points_to(phi.getShortCircuitInput(), context, value, cls, origin) - } - - /** Holds if the ESSA definition `def` refers to `(value, cls, origin)` given the context `context`. */ - predicate ssa_definition_points_to(EssaDefinition def, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - ssa_phi_points_to(def, context, value, cls, origin) - or - exists(ControlFlowNode orig | - ssa_node_definition_points_to(def, context, value, cls, orig) and - origin = CfgOrigin::fromCfgNode(orig) - ) - or - Filters::ssa_filter_definition_points_to(def, context, value, cls, origin) - or - ssa_node_refinement_points_to(def, context, value, cls, origin) - } - - pragma [nomagic] - private predicate ssa_node_definition_points_to_unpruned(EssaNodeDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - assignment_points_to(def, context, value, cls, origin) - or - parameter_points_to(def, context, value, cls, origin) - or - self_parameter_points_to(def, context, value, cls, origin) - or - delete_points_to(def, context, value, cls, origin) - or - module_name_points_to(def, context, value, cls, origin) - or - scope_entry_points_to(def, context, value, cls, origin) - or - implicit_submodule_points_to(def, context, value, cls, origin) - or - iteration_definition_points_to(def, context, value, cls, origin) - /* - * No points-to for non-local function entry definitions yet. - */ - } - - pragma [noinline] - private predicate reachable_definitions(EssaNodeDefinition def) { - Layer::reachableBlock(def.getDefiningNode().getBasicBlock(), _) - } - - pragma [noinline] - private predicate ssa_node_definition_points_to(EssaNodeDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - reachable_definitions(def) and - ssa_node_definition_points_to_unpruned(def, context, value, cls, origin) - } - - pragma [noinline] - private predicate ssa_node_refinement_points_to(EssaNodeRefinement def, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - method_callsite_points_to(def, context, value, cls, origin) - or - import_star_points_to(def, context, value, cls, origin) - or - attribute_assignment_points_to(def, context, value, cls, origin) - or - callsite_points_to(def, context, value, cls, origin) - or - argument_points_to(def, context, value, cls, origin) - or - attribute_delete_points_to(def, context, value, cls, origin) - or - Filters::uni_edged_phi_points_to(def, context, value, cls, origin) - } - - /** Points-to for normal assignments `def = ...`. */ - pragma [noinline] - private predicate assignment_points_to(AssignmentDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - points_to(def.getValue(), context, value, cls, origin) - } - - /** Helper for `parameter_points_to` */ - pragma [noinline] - private predicate positional_parameter_points_to(ParameterDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(PointsToContext caller, ControlFlowNode arg | - points_to(arg, caller, value, cls, origin) and - Flow::callsite_argument_transfer(arg, caller, def, context) - ) - or - not def.isSelf() and not def.getParameter().isVarargs() and not def.getParameter().isKwargs() and - context.isRuntime() and value = unknownValue() and cls = theUnknownType() and origin = def.getDefiningNode() - } - - /** Helper for `parameter_points_to` */ - pragma [noinline] - private predicate named_parameter_points_to(ParameterDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(CallNode call, PointsToContext caller, FunctionObject func, string name | - context.fromCall(call, func, caller) and - def.getParameter() = func.getFunction().getArgByName(name) and - points_to(call.getArgByName(name), caller, value, cls, origin) - ) - } - - /** Points-to for parameter. `def foo(param): ...`. */ - pragma [noinline] - private predicate parameter_points_to(ParameterDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - positional_parameter_points_to(def, context, value, cls, origin) - or - named_parameter_points_to(def, context, value, cls, origin) - or - default_parameter_points_to(def, context, value, cls, origin) - or - special_parameter_points_to(def, context, value, cls, origin) - } - - /** Helper for parameter_points_to */ - private predicate default_parameter_points_to(ParameterDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - default_value_points_to(def, value, cls, origin) and - context_for_default_value(def, context) - } - - /** Helper for default_parameter_points_to */ - pragma [noinline] - private predicate default_value_points_to(ParameterDefinition def, Object value, ClassObject cls, ControlFlowNode origin) { - exists(PointsToContext imp | imp.isImport() | points_to(def.getDefault(), imp, value, cls, origin)) - } - - /** Helper for default_parameter_points_to */ - pragma [noinline] - private predicate context_for_default_value(ParameterDefinition def, PointsToContext context) { - context.isRuntime() - or - exists(PointsToContext caller, CallNode call, FunctionObject func, int n | - context.fromCall(call, func, caller) and - func.getFunction().getArg(n) = def.getParameter() and - not exists(call.getArg(n)) and - not exists(call.getArgByName(def.getParameter().asName().getId())) and - not exists(call.getNode().getKwargs()) and - not exists(call.getNode().getStarargs()) - ) - } - - /** Helper for parameter_points_to */ - pragma [noinline] - private predicate special_parameter_points_to(ParameterDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - context.isRuntime() and - exists(ControlFlowNode param | - param = def.getDefiningNode() | - varargs_points_to(param, cls) and value = TupleObject::empty() and origin = param - or - varargs_points_to(param, cls) and value = param and origin = param - or - kwargs_points_to(param, cls) and value = param and origin = param - ) - or - exists(PointsToContext caller, CallNode call, FunctionObject func, Parameter p | - context.fromCall(call, func, caller) and - func.getFunction().getAnArg() = p and p = def.getParameter() and - not p.isSelf() and - not exists(call.getArg(p.getPosition())) and - not exists(call.getArgByName(p.getName())) and - (exists(call.getNode().getKwargs()) or exists(call.getNode().getStarargs())) and - value = unknownValue() and cls = theUnknownType() and origin = def.getDefiningNode() - ) - } - - /** Holds if the `(obj, caller)` pair matches up with `(self, callee)` pair across call. */ - pragma [noinline] - private predicate callsite_self_argument_transfer(EssaVariable obj, PointsToContext caller, ParameterDefinition self, PointsToContext callee) { - self.isSelf() and - exists(CallNode call, PyFunctionObject meth | - meth.getParameter(0) = self and - callee.fromCall(call, caller) | - Calls::plain_method_call(meth, caller, call) and - obj.getASourceUse() = call.getFunction().(AttrNode).getObject() - or - Calls::super_method_call(caller, call, obj, meth) - ) - } - - /** Points-to for self parameter: `def meth(self, ...): ...`. */ - pragma [noinline] - private predicate self_parameter_points_to(ParameterDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - def.isSelf() and - exists(FunctionObject meth, Function scope | - meth.getFunction() = scope | - def.getDefiningNode().getScope() = scope and - context.isRuntime() and context.appliesToScope(scope) and - scope.getScope() = cls.getPyClass() and - Types::concrete_class(cls) and - value = def.getDefiningNode() and origin = value and - /* We want to allow decorated functions, otherwise we lose a lot of useful information. - * However, we want to exclude any function whose arguments are permuted by the decorator. - * In general we can't do that, but we can special case the most common ones. - */ - neither_class_nor_static_method(scope) - ) - or - exists(EssaVariable obj, PointsToContext caller, CfgOrigin orig | - origin = orig.asCfgNodeOrHere(def.getDefiningNode()) and - ssa_variable_points_to(obj, caller, value, cls, orig) and - callsite_self_argument_transfer(obj, caller, def, context) - ) - or - cls_parameter_points_to(def, context, value, cls, origin) - } - - private predicate neither_class_nor_static_method(Function f) { - not exists(f.getADecorator()) - or - exists(ControlFlowNode deco | - deco = f.getADecorator().getAFlowNode() | - exists(Object o | - points_to(deco, _, o, _, _) | - o != theStaticMethodType() and - o != theClassMethodType() - ) - or not deco instanceof NameNode - ) - } - - - pragma [noinline] - private predicate cls_parameter_points_to(ParameterDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - def.isSelf() and - exists(CallNode call, PyFunctionObject meth, Object obj, ClassObject objcls, PointsToContext caller | - context.fromCall(call, caller) and - cls_method_object_points_to(call, caller, meth, obj, objcls, origin) and - def.getScope() = meth.getFunction() - | - obj instanceof ClassObject and value = obj and cls = objcls - or - obj.notClass() and value = objcls and cls = Types::class_get_meta_class(objcls) - ) - } - - /* Factor out part of `cls_parameter_points_to` to prevent bad join-order */ - pragma [noinline] - private predicate cls_method_object_points_to(CallNode call, PointsToContext context, PyFunctionObject meth, Object value, ClassObject cls, ControlFlowNode origin) { - exists(AttrNode attr | - class_method_call(_, attr, meth, context, call) and - points_to(attr.getObject(), context, value, cls, origin) - ) - } - - /** Points-to for deletion: `del name`. */ - pragma [noinline] - private predicate delete_points_to(DeletionDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - value = undefinedVariable() and cls = theUnknownType() and origin = def.getDefiningNode() and context.appliesToScope(def.getScope()) - } - - /** Implicit "definition" of the names of submodules at the start of an `__init__.py` file. - * - * PointsTo isn't exactly how the interpreter works, but is the best approximation we can manage statically. - */ - pragma [noinline] - private predicate implicit_submodule_points_to(ImplicitSubModuleDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - exists(PackageObject package | - package.getInitModule().getModule() = def.getDefiningNode().getScope() | - value = package.submodule(def.getSourceVariable().getName()) and - cls = theModuleType() and - origin = value and - context.isImport() - ) - } - - /** Implicit "definition" of `__name__` at the start of a module. */ - pragma [noinline] - private predicate module_name_points_to(ScopeEntryDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - def.getVariable().getName() = "__name__" and - exists(Module m | - m = def.getScope() - | - value = module_dunder_name(m) and context.isImport() - or - value = object_for_string("__main__") and context.isMain() and context.appliesToScope(m) - ) and - cls = theStrType() and origin = def.getDefiningNode() - } - - private Object module_dunder_name(Module m) { - exists(string name | - result = object_for_string(name) | - if m.isPackageInit() then - name = m.getPackage().getName() - else - name = m.getName() - ) - } - - /** Definition of iteration variable in loop */ - pragma [noinline] - private predicate iteration_definition_points_to(IterationDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - points_to(def.getSequence(), context, unknownValue(), _, _) and - value = unknownValue() and cls = theUnknownType() and origin = def.getDefiningNode() - } - - /** Points-to for implicit variable declarations at scope-entry. */ - pragma [noinline] - private predicate scope_entry_points_to(ScopeEntryDefinition def, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) { - /* Transfer from another scope */ - exists(EssaVariable var, PointsToContext outer, CfgOrigin orig | - Flow::scope_entry_value_transfer(var, outer, def, context) and - ssa_variable_points_to(var, outer, value, cls, orig) and - origin = orig.asCfgNodeOrHere(def.getDefiningNode()) - ) - or - /* Undefined variable */ - exists(Scope scope | - not def.getVariable().getName() = "__name__" and - not def.getVariable().getName() = "$" and - def.getScope() = scope and context.appliesToScope(scope) | - def.getSourceVariable() instanceof GlobalVariable and scope instanceof Module - or - def.getSourceVariable() instanceof LocalVariable and (context.isImport() or context.isRuntime() or context.isMain()) - ) and - value = undefinedVariable() and cls = theUnknownType() and origin = def.getDefiningNode() - or - /* Builtin not defined in outer scope */ - exists(Module mod, GlobalVariable var | - var = def.getSourceVariable() and - mod = def.getScope().getEnclosingModule() and - context.appliesToScope(def.getScope()) and - not exists(EssaVariable v | v.getSourceVariable() = var and v.getScope() = mod) and - builtin_name_points_to(var.getId(), value, cls) and origin = def.getDefiningNode() - ) - } - - /** Points-to for a variable (possibly) redefined by a call: - * `var = ...; foo(); use(var)` - * Where var may be redefined in call to `foo` if `var` escapes (is global or non-local). - */ - pragma [noinline] - private predicate callsite_points_to(CallsiteRefinement def, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - exists(SsaSourceVariable srcvar | - srcvar = def.getSourceVariable() | - if srcvar instanceof EscapingAssignmentGlobalVariable then ( - /* If global variable can be reassigned, we need to track it through calls */ - exists(EssaVariable var, PointsToContext callee | - Flow::callsite_exit_value_transfer(var, callee, def, context) and - ssa_variable_points_to(var, callee, value, cls, origin) - ) - or - callsite_points_to_python(def, context, value, cls, origin) - or - callsite_points_to_builtin(def, context, value, cls, origin) - ) else ( - /* Otherwise we can assume its value (but not those of its attributes or members) has not changed. */ - ssa_variable_points_to(def.getInput(), context, value, cls, origin) - ) - ) - } - - /** Holds if `call`, in `context` is a call to a function that does not modify the refined variable */ - private predicate call_to_safe_function(CallsiteRefinement def, PointsToContext context) { - exists(CallNode call | - call = def.getCall() and - context.untrackableCall(call) and - exists(PyFunctionObject modifier, Function f | - f = modifier.getFunction() and - call = get_a_call(modifier, context) and - not modifies_escaping_variable(f, def.getSourceVariable()) - ) - ) - } - - private predicate callsite_points_to_python(CallsiteRefinement def, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - exists(EssaVariable input | - input = def.getInput() and - ssa_variable_points_to(input, context, value, cls, origin) and - call_to_safe_function(def, context) - ) - } - - private predicate modifies_escaping_variable(Function modifier, PythonSsaSourceVariable var) { - exists(var.redefinedAtCallSite()) and - modifier.getBody().contains(var.(Variable).getAStore()) - } - - pragma [noinline] - private predicate callsite_points_to_builtin(CallsiteRefinement def, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - ssa_variable_points_to(def.getInput(), context, value, cls, origin) and - exists(CallNode call | - call = def.getCall() | - // An identifiable callee is a builtin - exists(BuiltinCallable opaque | get_a_call(opaque, _) = call) - ) - } - - /** Pass through for `self` for the implicit re-definition of `self` in `self.foo()`. */ - private predicate method_callsite_points_to(MethodCallsiteRefinement def, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - /* The value of self remains the same, only the attributes may change */ - ssa_variable_points_to(def.getInput(), context, value, cls, origin) - } - - /** Points-to for `from ... import *`. */ - private predicate import_star_points_to(ImportStarRefinement def, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - exists(CfgOrigin orig | - origin = orig.fix(def.getDefiningNode()) - | - exists(ModuleObject mod, string name | - Flow::module_and_name_for_import_star(mod, name, def, context) | - /* Attribute from imported module */ - module_exports(mod, name) and - Layer::module_attribute_points_to(mod, name, value, cls, orig) - ) - or - exists(EssaVariable var | - /* Retain value held before import */ - Flow::variable_not_redefined_by_import_star(var, context, def) and - ssa_variable_points_to(var, context, value, cls, orig) - ) - ) - } - - /** Attribute assignments have no effect as far as value tracking is concerned, except for `__class__`. */ - pragma [noinline] - private predicate attribute_assignment_points_to(AttributeAssignment def, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - if def.getName() = "__class__" then - ssa_variable_points_to(def.getInput(), context, value, _, _) and points_to(def.getValue(), _, cls, _,_) and - origin = CfgOrigin::fromCfgNode(def.getDefiningNode()) - else - ssa_variable_points_to(def.getInput(), context, value, cls, origin) - } - - /** Ignore the effects of calls on their arguments. PointsTo is an approximation, but attempting to improve accuracy would be very expensive for very little gain. */ - pragma [noinline] - private predicate argument_points_to(ArgumentRefinement def, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - ssa_variable_points_to(def.getInput(), context, value, cls, origin) - } - - /** Attribute deletions have no effect as far as value tracking is concerned. */ - pragma [noinline] - private predicate attribute_delete_points_to(EssaAttributeDeletion def, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - ssa_variable_points_to(def.getInput(), context, value, cls, origin) - } - - /* Data flow for attributes. These mirror the "normal" points-to predicates. - * For each points-to predicate `xxx_points_to(XXX def, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin)` - * There is an equivalent predicate that tracks the values in attributes: - * `xxx_named_attribute_points_to(XXX def, PointsToContext context, string name, Object value, ClassObject cls, ControlFlowNode origin)` - * */ - - /** INTERNAL -- Public for testing only. - * - * Hold if the attribute `name` of the ssa variable `var` refers to `(value, cls, origin)`. - */ - predicate ssa_variable_named_attribute_points_to(EssaVariable var, PointsToContext context, string name, Object value, ClassObject cls, CfgOrigin origin) { - ssa_definition_named_attribute_points_to(var.getDefinition(), context, name, value, cls, origin) - } - - /** Helper for `ssa_variable_named_attribute_points_to`. */ - private predicate ssa_definition_named_attribute_points_to(EssaDefinition def, PointsToContext context, string name, Object value, ClassObject cls, CfgOrigin origin) { - ssa_phi_named_attribute_points_to(def, context, name, value, cls, origin) - or - exists(ControlFlowNode orig | - ssa_node_definition_named_attribute_points_to(def, context, name, value, cls, orig) and - origin = CfgOrigin::fromCfgNode(orig) - ) - or - ssa_node_refinement_named_attribute_points_to(def, context, name, value, cls, origin) - or - Filters::ssa_filter_definition_named_attribute_points_to(def, context, name, value, cls, origin) - } - - /** Holds if the attribute `name` of the ssa phi-function definition `phi` refers to `(value, cls, origin)`. */ - pragma[noinline] - private predicate ssa_phi_named_attribute_points_to(PhiFunction phi, PointsToContext context, string name, Object value, ClassObject cls, CfgOrigin origin) { - ssa_variable_named_attribute_points_to(phi.getAnInput(), context, name, value, cls, origin) - } - - /** Helper for `ssa_definition_named_attribute_points_to`. */ - pragma[noinline] - private predicate ssa_node_definition_named_attribute_points_to(EssaNodeDefinition def, PointsToContext context, string name, Object value, ClassObject cls, ControlFlowNode origin) { - assignment_named_attribute_points_to(def, context, name, value, cls, origin) - or - delete_named_attribute_points_to(def, context, name, value, cls, origin) - or - self_parameter_named_attribute_points_to(def, context, name, value, cls, origin) - or - scope_entry_named_attribute_points_to(def, context, name, value, cls, origin) - } - - /** Helper for `ssa_definition_named_attribute_points_to`. */ - pragma[noinline] - private predicate ssa_node_refinement_named_attribute_points_to(EssaNodeRefinement def, PointsToContext context, string name, Object value, ClassObject cls, CfgOrigin origin) { - attribute_assignment_named_attribute_points_to(def, context, name, value, cls, origin) - or - attribute_delete_named_attribute_points_to(def, context, name, value, cls, origin) - or - import_star_named_attribute_points_to(def, context, name, value, cls, origin) - or - self_callsite_named_attribute_points_to(def, context, name, value, cls, origin) - or - argument_named_attribute_points_to(def, context, name, value, cls, origin) - } - - pragma[noinline] - private predicate scope_entry_named_attribute_points_to(ScopeEntryDefinition def, PointsToContext context, string name, Object value, ClassObject cls, ControlFlowNode origin) { - exists(EssaVariable var, PointsToContext outer, CfgOrigin orig | - Flow::scope_entry_value_transfer(var, outer, def, context) and - ssa_variable_named_attribute_points_to(var, outer, name, value, cls, orig) and - origin = orig.asCfgNodeOrHere(def.getDefiningNode()) - ) - or - origin = def.getDefiningNode() and - isModuleStateVariable(def.getVariable()) and - context.isImport() and - exists(PackageObject package | - package.getInitModule().getModule() = def.getScope() | - explicitly_imported(package.submodule(name)) and - value = undefinedVariable() and - cls = theUnknownType() - ) - } - - pragma[noinline] - private predicate assignment_named_attribute_points_to(AssignmentDefinition def, PointsToContext context, string name, Object value, ClassObject cls, ControlFlowNode origin) { - exists(CfgOrigin orig | - named_attribute_points_to(def.getValue(), context, name, value, cls, orig) and - origin = orig.asCfgNodeOrHere(def.getDefiningNode()) - ) - } - - pragma[noinline] - private predicate attribute_assignment_named_attribute_points_to(AttributeAssignment def, PointsToContext context, string name, Object value, ClassObject cls, CfgOrigin origin) { - points_to(def.getValue(), context, value, cls, origin.toCfgNode()) and name = def.getName() - or - ssa_variable_named_attribute_points_to(def.getInput(), context, name, value, cls, origin) and not name = def.getName() - } - - /** Holds if `def` defines the attribute `name`. - * - * `def` takes the form `setattr(use, "name")` where `use` is the input to the definition. - */ - private boolean sets_attribute(ArgumentRefinement def, string name) { - exists(ControlFlowNode func, Object obj | - two_args_first_arg_string(def, func, name) and - points_to(func, _, obj, _, _) | - obj = Object::builtin("setattr") and result = true - or - obj != Object::builtin("setattr") and result = false - ) - } - - private predicate two_args_first_arg_string(ArgumentRefinement def, ControlFlowNode func, string name) { - exists(CallNode call | - call = def.getDefiningNode() and - call.getFunction() = func and - def.getInput().getAUse() = call.getArg(0) and - call.getArg(1).getNode().(StrConst).getText() = name - ) - } - - pragma[noinline] - private predicate argument_named_attribute_points_to(ArgumentRefinement def, PointsToContext context, string name, Object value, ClassObject cls, CfgOrigin origin) { - not two_args_first_arg_string(def, _, name) and ssa_variable_named_attribute_points_to(def.getInput(), context, name, value, cls, origin) - or - sets_attribute(def, name) = true and points_to(def.getDefiningNode().(CallNode).getArg(2), context, value, cls, origin.toCfgNode()) - or - sets_attribute(def, name) = false and ssa_variable_named_attribute_points_to(def.getInput(), context, name, value, cls, origin) - } - - /** Holds if the self variable in the callee (`(var, callee)`) refers to the same object as `def` immediately after the call, (`(def, caller)`). */ - pragma[noinline] - private predicate callee_self_variable(EssaVariable var, PointsToContext callee, SelfCallsiteRefinement def, PointsToContext caller) { - exists(FunctionObject func, LocalVariable self | - callee.fromCall(def.getCall(), func, caller) and - BaseFlow::reaches_exit(var) and - var.getSourceVariable() = self and - self.isSelf() and - self.getScope() = func.getFunction() - ) - } - - pragma[noinline] - private predicate self_callsite_named_attribute_points_to(SelfCallsiteRefinement def, PointsToContext context, string name, Object value, ClassObject cls, CfgOrigin origin) { - exists(EssaVariable var, PointsToContext callee | - ssa_variable_named_attribute_points_to(var, callee, name, value, cls, origin) and - callee_self_variable(var, callee, def, context) - ) - } - - /** Gets the (temporally) preceding variable for `self`, e.g. `def` is in method `foo()` and `result` is in `__init__()`. */ - private EssaVariable preceding_self_variable(ParameterDefinition def) { - def.isSelf() and - exists(Function preceding, Function method | - method = def.getScope() and - // Only methods - preceding.isMethod() and preceding.precedes(method) and - BaseFlow::reaches_exit(result) and result.getSourceVariable().(Variable).isSelf() and - result.getScope() = preceding - ) - } - - /** Maps the caller object/context to callee parameter/context for self in calls to methods */ - private predicate self_in_method_call(ControlFlowNode obj, PointsToContext caller, ParameterDefinition self, PointsToContext callee) { - self.isSelf() and - exists(FunctionObject meth, CallNode call | - meth.getFunction() = self.getScope() and - callee.fromCall(call, meth, caller) and - call.getFunction().(AttrNode).getObject() = obj - ) - } - - pragma [noinline] - private predicate self_parameter_named_attribute_points_to(ParameterDefinition def, PointsToContext context, string name, Object value, ClassObject vcls, ControlFlowNode origin) { - exists(CfgOrigin orig | - origin = orig.toCfgNode() - | - context.isRuntime() and executes_in_runtime_context(def.getScope()) and - ssa_variable_named_attribute_points_to(preceding_self_variable(def), context, name, value, vcls, orig) - or - exists(PointsToContext caller_context, ControlFlowNode obj | - self_in_method_call(obj, caller_context, def, context) and - named_attribute_points_to(obj, caller_context, name, value, vcls, orig) - ) - ) - } - - private predicate delete_named_attribute_points_to(DeletionDefinition def, PointsToContext context, string name, Object value, ClassObject cls, ControlFlowNode origin) { - none() - } - - private predicate attribute_delete_named_attribute_points_to(EssaAttributeDeletion def, PointsToContext context, string name, Object value, ClassObject cls, CfgOrigin origin) { - none() - } - - /* Helper for import_star_named_attribute_points_to */ - pragma [noinline] - private predicate star_variable_import_star_module(ImportStarRefinement def, ImportStarNode imp, PointsToContext context, ModuleObject mod) { - isModuleStateVariable(def.getVariable()) and - exists(ControlFlowNode fmod | - fmod = imp.getModule() and - imp = def.getDefiningNode() and - points_to(fmod, context, mod, _, _) - ) - } - - /* Helper for import_star_named_attribute_points_to */ - pragma [noinline, nomagic] - private predicate ssa_star_variable_input_points_to(ImportStarRefinement def, PointsToContext context, string name, Object value, ClassObject cls, CfgOrigin origin) { - exists(EssaVariable var | - ssa_star_import_star_input(def, var) and - ssa_variable_named_attribute_points_to(var, context, name, value, cls, origin) - ) - } - - /* Helper for ssa_star_variable_input_points_to */ - pragma [noinline] - private predicate ssa_star_import_star_input(ImportStarRefinement def, EssaVariable var) { - isModuleStateVariable(def.getVariable()) and var = def.getInput() - } - - pragma [noinline] - private predicate import_star_named_attribute_points_to(ImportStarRefinement def, PointsToContext context, string name, Object value, ClassObject cls, CfgOrigin origin) { - exists(ImportStarNode imp, ModuleObject mod, CfgOrigin orig | - origin = orig.fix(imp) and - star_variable_import_star_module(def, imp, context, mod) | - /* Attribute from imported module */ - module_exports_boolean(mod, name) = true and - Layer::module_attribute_points_to(mod, name, value, cls, orig) and - not exists(Variable v | v.getId() = name and v.getScope() = imp.getScope()) - or - /* Retain value held before import */ - module_exports_boolean(mod, name) = false and - ssa_star_variable_input_points_to(def, context, name, value, cls, orig) - ) - } - - } - - private module Filters { - - /** Holds if `expr` is the operand of a unary `not` expression. */ - private ControlFlowNode not_operand(ControlFlowNode expr) { - expr.(UnaryExprNode).getNode().getOp() instanceof Not and - result = expr.(UnaryExprNode).getOperand() - } - - /** Gets the value that `expr` evaluates to (when converted to a boolean) when `use` refers to `(val, cls, origin)` - * and `expr` contains `use` and both are contained within a test. */ - pragma [nomagic] - boolean evaluates_boolean(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, Object val, ClassObject cls, ControlFlowNode origin) { - result = isinstance_test_evaluates_boolean(expr, use, context, val, cls, origin) - or - result = issubclass_test_evaluates_boolean(expr, use, context, val, cls, origin) - or - result = equality_test_evaluates_boolean(expr, use, context, val, cls, origin) - or - result = callable_test_evaluates_boolean(expr, use, context, val, cls, origin) - or - result = hasattr_test_evaluates_boolean(expr, use, context, val, cls, origin) - or - result = evaluates(not_operand(expr), use, context, val, cls, origin).booleanNot() - } - - /** Gets the value that `expr` evaluates to (when converted to a boolean) when `use` refers to `(val, cls, origin)` - * and `expr` contains `use` and both are contained within a test. */ - pragma [nomagic] - boolean evaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, Object val, ClassObject cls, ControlFlowNode origin) { - result = evaluates_boolean(expr, use, context, val, cls, origin) - or - result = true and evaluates_int(expr, use, context, val, cls, origin) != 0 - or - result = false and evaluates_int(expr, use, context, val, cls, origin) = 0 - or - result = truth_test_evaluates_boolean(expr, use, context, val, cls, origin) - } - - private boolean maybe() { - result = true or result = false - } - - pragma [nomagic] - private boolean issubclass_test_evaluates_boolean(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, Object val, ClassObject cls, ControlFlowNode origin) { - points_to(use, context, val, cls, origin) and - exists(ControlFlowNode clsNode | - BaseFilters::issubclass(expr, clsNode, use) - | - exists(ClassObject scls | - result = Types::is_improper_subclass_bool(val, scls) - | - points_to(clsNode, context, scls, _, _) - or - element_of_points_to_tuple(clsNode, context, scls) and result = true - ) - or - val = unknownValue() and result = maybe() - or - val = theUnknownType() and result = maybe() - ) - } - - pragma [nomagic] - private boolean isinstance_test_evaluates_boolean(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, Object val, ClassObject cls, ControlFlowNode origin) { - points_to(use, context, val, cls, origin) and - exists(ControlFlowNode clsNode | - BaseFilters::isinstance(expr, clsNode, use) - | - exists(ClassObject scls | - result = Types::is_improper_subclass_bool(cls, scls) - | - points_to(clsNode, context, scls, _, _) - or - element_of_points_to_tuple(clsNode, context, scls) and result = true - ) - or - val = unknownValue() and result = maybe() - ) - } - - pragma [noinline] - private boolean equality_test_evaluates_boolean(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, Object val, ClassObject cls, ControlFlowNode origin) { - exists(ControlFlowNode l, ControlFlowNode r, boolean sense | - contains_interesting_expression_within_test(expr, use) and - BaseFilters::equality_test(expr, l, sense, r) | - exists(int il, int ir | - il = evaluates_int(l, use, context, val, cls, origin) and ir = simple_int_value(r) - | - result = sense and il = ir - or - result = sense.booleanNot() and il != ir - ) - or - use = l and - exists(Object other | - /* Must be discrete values, not just types of things */ - equatable_value(val) and equatable_value(other) and - points_to(use, context, val, cls, origin) and - points_to(r, context, other, _, _) | - other != val and result = sense.booleanNot() - or - other = val and result = sense - ) - ) - } - - pragma [noinline] - private boolean truth_test_evaluates_boolean(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, Object val, ClassObject cls, ControlFlowNode origin) { - contains_interesting_expression_within_test(expr, use) and - points_to(use, context, val, cls, origin) and - expr = use and - ( - val.booleanValue() = result - or - Types::instances_always_true(cls) and result = true - or - val.maybe() and result = true - or - val.maybe() and result = false - ) - } - - pragma [noinline] - private boolean callable_test_evaluates_boolean(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, Object val, ClassObject cls, ControlFlowNode origin) { - contains_interesting_expression_within_test(expr, use) and - points_to(use, context, val, cls, origin) and - BaseFilters::is_callable(expr, use) and - ( - result = Types::class_has_attribute_bool(cls, "__call__") - or - cls = theUnknownType() and result = maybe() - ) - } - - pragma [noinline] - private boolean hasattr_test_evaluates_boolean(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, Object val, ClassObject cls, ControlFlowNode origin) { - contains_interesting_expression_within_test(expr, use) and - points_to(use, context, val, cls, origin) and - exists(string name | - BaseFilters::hasattr(expr, use, name) | - result = Types::class_has_attribute_bool(cls, name) - ) - } - - /** Holds if meaningful equality tests can be made with `o`. - * True for basic objects like 3 or None, but it is also true for sentinel objects. - */ - predicate equatable_value(Object o) { - comparable_value(o) - or - o.(ControlFlowNode).getScope() instanceof Module and - exists(ClassObject c | - c.isBuiltin() and - points_to(o.(CallNode).getFunction(), _, c, _, _) - ) - } - - /** Holds if meaningful comparisons can be made with `o`. - * True for basic objects like 3 or None. - */ - predicate comparable_value(Object o) { - o.isBuiltin() and not o = unknownValue() and not o = undefinedVariable() - or - exists(o.booleanValue()) - } - - - /** Holds if the test on `use` is a test that we can potentially understand */ - private predicate comprehensible_test(ControlFlowNode test, ControlFlowNode use) { - BaseFilters::issubclass(test, _, use) - or - BaseFilters::isinstance(test, _, use) - or - BaseFilters::equality_test(test, use, _, _) - or - exists(ControlFlowNode l | - BaseFilters::equality_test(test, l, _, _) | - literal_or_len(l) - ) - or - BaseFilters::is_callable(test, use) - or - BaseFilters::hasattr(test, use, _) - or - test = use - or - literal_or_len(test) - or - comprehensible_test(not_operand(test), use) - } - - - /** Gets the simple integer value of `f` for numeric literals. */ - private int simple_int_value(ControlFlowNode f) { - exists(NumericObject num | - points_to(f, _, num, _, _) and - result = num.intValue() - ) - } - - /** Gets the integer value that `expr` evaluates to given that `use` refers to `val` and `use` is a part of `expr`. - * Only applies to numeric literal and `len()` of sequences. */ - pragma [noinline] - private int evaluates_int(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, Object val, ClassObject cls, ControlFlowNode origin) { - contains_interesting_expression_within_test(expr, use) and - points_to(use, context, val, cls, origin) and - ( - exists(CallNode call | - call = expr and - points_to(call.getFunction(), context, Object::builtin("len"), _, _) and - use = call.getArg(0) and - val.(SequenceObject).getLength() = result - ) - or - expr = use and result = val.(NumericObject).intValue() - ) - } - - private predicate literal_or_len(ControlFlowNode expr) { - expr.getNode() instanceof Num - or - expr.(CallNode).getFunction().(NameNode).getId() = "len" - } - - /** Holds if ESSA edge refinement, `def`, refers to `(value, cls, origin)`. */ - predicate ssa_filter_definition_points_to(PyEdgeRefinement def, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - exists(ControlFlowNode test, ControlFlowNode use | - refinement_test(test, use, test_evaluates_boolean(test, use, context, value, cls, origin.toCfgNode()), def) - ) - or - /* If we can't understand the test, assume that value passes through. - * Or, if the value is `unknownValue()` then let it pass through as well. */ - exists(ControlFlowNode test, ControlFlowNode use | - refinement_test(test, use, _, def) and - ssa_variable_points_to(def.getInput(), context, value, cls, origin) | - not comprehensible_test(test, use) or - value = unknownValue() - ) - } - - /** Holds if ESSA definition, `uniphi`, refers to `(value, cls, origin)`. */ - pragma [noinline] - predicate uni_edged_phi_points_to(SingleSuccessorGuard uniphi, PointsToContext context, Object value, ClassObject cls, CfgOrigin origin) { - exists(ControlFlowNode test, ControlFlowNode use | - /* Because calls such as `len` may create a new variable, we need to go via the source variable - * That is perfectly safe as we are only dealing with calls that do not mutate their arguments. - */ - use = uniphi.getInput().getSourceVariable().(Variable).getAUse() and - test = uniphi.getDefiningNode() and - uniphi.getSense() = test_evaluates_boolean(test, use, context, value, cls, origin.toCfgNode()) - ) - } - - /** Holds if the named attibute of ESSA edge refinement, `def`, refers to `(value, cls, origin)`. */ - pragma[noinline] - predicate ssa_filter_definition_named_attribute_points_to(PyEdgeRefinement def, PointsToContext context, string name, Object value, ClassObject cls, CfgOrigin origin) { - exists(ControlFlowNode test, AttrNode use, boolean sense | - edge_refinement_attr_use_sense(def, test, use, name, sense) and - sense = test_evaluates_boolean(test, use, context, value, cls, origin.toCfgNode()) - ) - or - exists(EssaVariable input | - input = def.getInput() and - not edge_refinement_test(def, input, name) and - SSA::ssa_variable_named_attribute_points_to(input, context, name, value, cls, origin) - ) - } - - /* Helper for ssa_filter_definition_named_attribute_points_to - * Holds if `use` is of the form `var.name` in the test of `def`, and `var` is the source variable of `def`, and `def` has sense `sense`. - */ - pragma[noinline] - private predicate edge_refinement_attr_use_sense(PyEdgeRefinement def, ControlFlowNode test, AttrNode use, string name, boolean sense) { - def.getSense() = sense and - exists(EssaVariable input | - input = def.getInput() and - test = def.getTest() and - use.getObject(name) = def.getInput().getSourceVariable().(Variable).getAUse() and - test_contains(test, use) - ) - } - - /* Helper for ssa_filter_definition_named_attribute_points_to */ - pragma[noinline] - private predicate edge_refinement_test(PyEdgeRefinement def, EssaVariable input, string name) { - exists(ControlFlowNode test | - input = def.getInput() and - test = def.getTest() | - exists(AttrNode use | - refinement_test(test, use.getObject(name), _, def) - ) - ) - } - - } - - cached module Types { - - /** INTERNAL -- Use `ClassObject.getBaseType(n)` instead. - * - * Gets the nth base class of the class. */ - cached Object class_base_type(ClassObject cls, int n) { - not result = unknownValue() and - exists(ClassExpr cls_expr | cls.getOrigin() = cls_expr | - points_to(cls_expr.getBase(n).getAFlowNode(), _, result, _, _) - or - is_new_style(cls) and not exists(cls_expr.getBase(0)) and result = theObjectType() and n = 0 - ) - or - result.asBuiltin() = cls.asBuiltin().getBaseClass() and n = 0 - or - cls = theUnknownType() and result = theObjectType() and n = 0 - } - - private Object py_base_type(ClassObject cls, int n) { - not result = unknownValue() and - exists(ClassExpr cls_expr | cls.getOrigin() = cls_expr | - points_to(cls_expr.getBase(n).getAFlowNode(), _, result, _, _) - ) - } - - cached int class_base_count(ClassObject cls) { - exists(ClassExpr cls_expr | - cls.getOrigin() = cls_expr | - result = strictcount(cls_expr.getABase()) - or - is_new_style_bool(cls) = true and not exists(cls_expr.getBase(0)) and result = 1 - or - is_new_style_bool(cls) = false and not exists(cls_expr.getBase(0)) and result = 0 - ) - or - cls = theObjectType() and result = 0 - or - exists(cls.asBuiltin().getBaseClass()) and cls != theObjectType() and result = 1 - or - cls = theUnknownType() and result = 1 - } - - /** INTERNAL -- Do not not use. - * - * Holds if a call to this class will return an instance of this class. - */ - cached predicate callToClassWillReturnInstance(ClassObject cls) { - callToClassWillReturnInstance(cls, 0) and - not callToPythonClassMayNotReturnInstance(cls.getPyClass()) - } - - private predicate callToClassWillReturnInstance(ClassObject cls, int n) { - n = class_base_count(cls) - or - callToClassWillReturnInstance(cls, n+1) and - exists(ClassObject base | - base = class_base_type(cls, n) | - /* Most builtin types "declare" `__new__`, such as `int`, yet are well behaved. */ - base.isBuiltin() - or - exists(Class c | - c = cls.getPyClass() and - not callToPythonClassMayNotReturnInstance(c) - ) - ) - } - - private predicate callToPythonClassMayNotReturnInstance(Class cls) { - /* Django does this, so we need to account for it */ - exists(Function init, LocalVariable self | - /* `self.__class__ = ...` in the `__init__` method */ - cls.getInitMethod() = init and - self.isSelf() and self.getScope() = init and - exists(AttrNode a | a.isStore() and a.getObject("__class__") = self.getAUse()) - ) - or - exists(Function new | new.getName() = "__new__" and new.getScope() = cls) - } - - cached boolean is_new_style_bool(ClassObject cls) { - major_version() = 3 and result = true - or - cls.isBuiltin() and result = true - or - get_an_improper_super_type(class_get_meta_class(cls)) = theTypeType() and result = true - or - class_get_meta_class(cls) = theClassType() and result = false - } - - /** INTERNAL -- Use `ClassObject.isNewStyle()` instead. */ - cached predicate is_new_style(ClassObject cls) { - is_new_style_bool(cls) = true - or - is_new_style(get_a_super_type(cls)) - } - - /** INTERNAL -- Use `ClassObject.getASuperType()` instead. */ - cached ClassObject get_a_super_type(ClassObject cls) { - result = class_base_type(cls, _) - or - result = class_base_type(get_a_super_type(cls), _) - } - - /** INTERNAL -- Use `ClassObject.getAnImproperSuperType()` instead. */ - cached ClassObject get_an_improper_super_type(ClassObject cls) { - result = cls - or - result = get_a_super_type(cls) - } - - cached boolean is_subclass_bool(ClassObject cls, ClassObject sup) { - if abcSubclass(cls, sup) then ( - /* Hard-code some abc subclass pairs -- In future we may change this to use stubs. */ - result = true - ) else ( - sup = class_base_type(cls, _) and result = true - or - is_subclass_bool(class_base_type(cls, _), sup) = true and result = true - or - result = is_subclass_bool(cls, sup, 0) - ) - } - - private predicate abcSubclass(ClassObject cls, ClassObject sup) { - cls = theListType() and sup = collectionsAbcClass("Iterable") - or - cls = theSetType() and sup = collectionsAbcClass("Iterable") - or - cls = theDictType() and sup = collectionsAbcClass("Iterable") - or - cls = theSetType() and sup = collectionsAbcClass("Set") - or - cls = theListType() and sup = collectionsAbcClass("Sequence") - or - cls = theDictType() and sup = collectionsAbcClass("Mapping") - } - - cached boolean is_improper_subclass_bool(ClassObject cls, ClassObject sup) { - result = is_subclass_bool(cls, sup) - or - result = true and cls = sup - } - - private boolean is_subclass_bool(ClassObject cls, ClassObject sup, int n) { - relevant_subclass_relation(cls, sup) and - ( - n = class_base_count(cls) and result = false and not cls = sup - or - exists(ClassObject basetype | - basetype = class_base_type(cls, n) | - not basetype = sup and - result = is_subclass_bool(cls, sup, n+1).booleanOr(is_subclass_bool(basetype, sup)) - or - basetype = sup and result = true - ) - ) - } - - private predicate relevant_subclass_relation(ClassObject cls, ClassObject sup) { - exists(ControlFlowNode supnode | - points_to(supnode, _, sup, _, _) - or - element_of_points_to_tuple(supnode, _, sup) - | - subclass_test(supnode, cls) - ) - or - exists(ClassObject sub | - relevant_subclass_relation(sub, sup) and - class_base_type(sub, _) = cls - ) - } - - /** Holds if there is a subclass test of `f` against class `cls`. - * Helper for relevant_subclass_relation. - */ - private predicate subclass_test(ControlFlowNode f, ClassObject cls) { - exists(ControlFlowNode use | - BaseFilters::issubclass(_, f, use) and points_to(use, _, cls, _, _) - or - BaseFilters::isinstance(_, f, use) and points_to(use, _, _, cls, _) - ) - } - - cached ClassList get_mro(ClassObject cls) { - result = new_style_mro(cls) and is_new_style_bool(cls) = true - or - result = old_style_mro(cls) and is_new_style_bool(cls) = false - } - - /** INTERNAL -- Use `ClassObject.declaredAttribute(name). instead. */ - cached predicate class_declared_attribute(ClassObject owner, string name, Object value, ClassObject vcls, CfgOrigin origin) { - /* Note that src_var must be a local variable, we aren't interested in the value that any global variable may hold */ - value != undefinedVariable() and - exists(EssaVariable var, LocalVariable src_var | - var.getSourceVariable() = src_var and - src_var.getId() = name and - var.getAUse() = owner.getImportTimeScope().getANormalExit() | - ssa_variable_points_to(var, _, value, vcls, origin) - ) - or - value.asBuiltin() = owner.asBuiltin().getMember(name) and class_declares_attribute(owner, name) and - origin = CfgOrigin::unknown() and vcls.asBuiltin() = value.asBuiltin().getClass() - } - - private predicate interesting_class_attribute(ClassList mro, string name) { - exists(ControlFlowNode use, ClassObject cls | - mro = cls.getMro() and - BaseFilters::hasattr(_, use, name) | - points_to(use, _, cls, _, _) or - points_to(use, _, _, cls, _) - ) - or - exists(ClassList sublist | - sublist.getTail() = mro and - interesting_class_attribute(sublist, name) - ) - or - name = "__call__" - } - - private predicate does_not_have_attribute(ClassList mro, string name) { - interesting_class_attribute(mro, name) and - ( - mro.isEmpty() - or - exists(ClassObject head, ClassList tail | - head = mro.getHead() and tail = mro.getTail() | - does_not_have_attribute(tail, name) and - not class_declares_attribute(head, name) - ) - ) - } - - /** Holds if the class `cls` has an attribute called `name` */ - cached predicate class_has_attribute(ClassObject cls, string name) { - class_declares_attribute(get_an_improper_super_type(cls), name) - } - - /** Gets `true` if the class `cls` is known to have attribute `name`, - * or `false` if the class `cls` is known to not have attribute `name`. - */ - cached boolean class_has_attribute_bool(ClassObject cls, string name) { - exists(ClassList mro | - mro = cls.getMro() | - mro.declares(name) and result = true - or - does_not_have_attribute(mro, name) and result = false - ) - } - - /** INTERNAL -- Use `ClassObject.attributeRefersTo(name, value, vlcs, origin). instead. - */ - cached predicate class_attribute_lookup(ClassObject cls, string name, Object value, ClassObject vcls, CfgOrigin origin) { - exists(ClassObject defn | - defn = get_mro(cls).findDeclaringClass(name) and - class_declared_attribute(defn, name, value, vcls, origin) - ) - } - - /** INTERNAL -- Use `ClassObject.failedInference(reason). instead. - * - * Holds if type inference failed to compute the full class hierarchy for this class for the reason given. */ - cached predicate failed_inference(ClassObject cls, string reason) { - strictcount(cls.getPyClass().getADecorator()) > 1 and reason = "Multiple decorators" - or - exists(cls.getPyClass().getADecorator()) and not six_add_metaclass(_, cls, _) and reason = "Decorator not understood" - or - exists(int i | - exists(((ClassExpr)cls.getOrigin()).getBase(i)) and reason = "Missing base " + i - | - not exists(class_base_type(cls, i)) - ) - or - exists(cls.getPyClass().getMetaClass()) and not exists(class_get_meta_class(cls)) and reason = "Failed to infer metaclass" - or - exists(int i | failed_inference(class_base_type(cls, i), _) and reason = "Failed inference for base class at position " + i) - or - exists(int i, Object base1, Object base2 | - base1 = class_base_type(cls, i) and - base2 = class_base_type(cls, i) and - base1 != base2 and - reason = "Multiple bases at position " + i - ) - or - exists(int i, int j | class_base_type(cls, i) = class_base_type(cls, j) and i != j and reason = "Duplicate bases classes") - or - cls = theUnknownType() and reason = "Unknown Type" - } - - /** INTERNAL -- Use `ClassObject.getMetaClass()` instead. - * - * Gets the metaclass for this class */ - cached ClassObject class_get_meta_class(ClassObject cls) { - result = declared_meta_class(cls) - or - has_declared_metaclass(cls) = false and result = get_inherited_metaclass(cls) - or - cls = theUnknownType() and result = theUnknownType() - } - - private ClassObject declared_meta_class(ClassObject cls) { - exists(Object obj | - ssa_variable_points_to(metaclass_var(cls.getPyClass()), _, obj, _, _) | - result = obj - or - obj = unknownValue() and result = theUnknownType() - ) - or - exists(Builtin meta | - result.asBuiltin() = meta and - meta = cls.asBuiltin().getClass() and - meta.inheritsFromType() - ) - or - exists(ControlFlowNode meta | - Types::six_add_metaclass(_, cls, meta) and - points_to(meta, _, result, _, _) - ) - } - - private boolean has_metaclass_var_metaclass(ClassObject cls) { - exists(Object obj | - ssa_variable_points_to(metaclass_var(cls.getPyClass()), _, obj, _, _) | - obj = undefinedVariable() and result = false - or - obj != undefinedVariable() and result = true - ) - or - exists(Class pycls | - pycls = cls.getPyClass() and - not exists(metaclass_var(pycls)) and result = false - ) - } - - private boolean has_declared_metaclass(ClassObject cls) { - exists(cls.asBuiltin().getClass()) and result = true - or - result = has_six_add_metaclass(cls).booleanOr(has_metaclass_var_metaclass(cls)) - } - - private EssaVariable metaclass_var(Class cls) { - result.getASourceUse() = cls.getMetaClass().getAFlowNode() - or - major_version() = 2 and not exists(cls.getMetaClass()) and - result.getName() = "__metaclass__" and - cls.(ImportTimeScope).entryEdge(result.getAUse(), _) - } - - private ClassObject get_inherited_metaclass(ClassObject cls) { - result = get_inherited_metaclass(cls, 0) - or - // Best guess if base is not a known class - exists(Object base | - base = class_base_type(cls, _) and - result = theUnknownType() | - base.notClass() - or - base = theUnknownType() - ) - } - - private ClassObject get_inherited_metaclass(ClassObject cls, int n) { - exists(Class c | - c = cls.getPyClass() and - n = count(c.getABase()) - | - major_version() = 3 and result = theTypeType() - or - major_version() = 2 and result = theClassType() - ) - or - exists(ClassObject meta1, ClassObject meta2 | - meta1 = class_get_meta_class(py_base_type(cls, n)) and - meta2 = get_inherited_metaclass(cls, n+1) - | - /* Choose sub-class */ - get_an_improper_super_type(meta1) = meta2 and result = meta1 - or - get_an_improper_super_type(meta2) = meta1 and result = meta2 - or - /* Choose new-style meta-class over old-style */ - meta2 = theClassType() and result = meta1 - or - /* Make sure we have a metaclass, even if base is unknown */ - meta1 = theUnknownType() and result = theTypeType() - or - meta2 = theUnknownType() and result = meta1 - ) - } - - private Object six_add_metaclass_function() { - exists(Module six, FunctionExpr add_metaclass | - add_metaclass.getInnerScope().getName() = "add_metaclass" and - add_metaclass.getScope() = six and - result.getOrigin() = add_metaclass - ) - } - - private ControlFlowNode decorator_call_callee(ClassObject cls) { - exists(CallNode decorator_call, CallNode decorator | - decorator_call.getArg(0) = cls and - decorator = decorator_call.getFunction() and - result = decorator.getFunction() - ) - } - - /** INTERNAL -- Do not use */ - cached boolean has_six_add_metaclass(ClassObject cls) { - exists(ControlFlowNode callee, Object func | - callee = decorator_call_callee(cls) and - points_to(callee, _, func, _, _) | - func = six_add_metaclass_function() and result = true - or - not func = six_add_metaclass_function() and result = false - ) - or - not exists(six_add_metaclass_function()) and result = false - or - not exists(decorator_call_callee(cls)) and result = false - } - - /** INTERNAL -- Do not use */ - cached predicate six_add_metaclass(CallNode decorator_call, ClassObject decorated, ControlFlowNode metaclass) { - exists(CallNode decorator | - decorator_call.getArg(0) = decorated and - decorator = decorator_call.getFunction() and - decorator.getArg(0) = metaclass | - points_to(decorator.getFunction(), _, six_add_metaclass_function(), _, _) - or - exists(ModuleObject six | - six.getName() = "six" and - points_to(decorator.getFunction().(AttrNode).getObject("add_metaclass"), _, six, _, _) - ) - ) - } - - /** INTERNAL -- Use `not cls.isAbstract()` instead. */ - cached predicate concrete_class(ClassObject cls) { - Types::class_get_meta_class(cls) != theAbcMetaClassObject() - or - exists(Class c | - c = cls.getPyClass() and - not exists(c.getMetaClass()) - | - forall(Function f | - f.getScope() = c | - not exists(Raise r, Name ex | - r.getScope() = f and - (r.getException() = ex or r.getException().(Call).getFunc() = ex) and - (ex.getId() = "NotImplementedError" or ex.getId() = "NotImplemented") - ) - ) - ) - } - - /** Holds if instances of class `cls` are always truthy. */ - cached predicate instances_always_true(ClassObject cls) { - cls = theObjectType() - or - instances_always_true(cls, 0) and - not exists(string meth | - class_declares_attribute(cls, meth) | - meth = "__bool__" or meth = "__len__" or - meth = "__nonzero__" and major_version() = 2 - ) - } - - /** Holds if instances of class `cls` are always truthy. */ - cached predicate instances_always_true(ClassObject cls, int n) { - not cls = theNoneType() and - n = class_base_count(cls) - or - instances_always_true(cls, n+1) and - instances_always_true(class_base_type(cls, n)) - } - - } - - /** Get the ESSA pseudo-variable used to retain module state - * during module initialization. Module attributes are handled - * as attributes of this variable, allowing the SSA form to track - * mutations of the module during its creation. - */ - private predicate isModuleStateVariable(EssaVariable var) { - var.getName() = "$" and var.getScope() instanceof Module - } - - /** INTERNAL -- Public for testing only */ - module Test { - - import Calls - import SSA - import Layer - - } - -} - -/* Helper classes for `super` dispatching. */ - -class SuperCall extends Object { - - EssaVariable self; - ClassObject start; - - override string toString() { - result = "super()" - } - - SuperCall() { - exists(CallNode call, PointsToContext context | - call = this and - PointsTo::points_to(call.getFunction(), _, theSuperType(), _, _) | - PointsTo::points_to(call.getArg(0), context, start, _, _) and - self.getASourceUse() = call.getArg(1) - or - major_version() = 3 and - not exists(call.getArg(0)) and - exists(Function func | - call.getScope() = func and - context.appliesToScope(func) and - /* Implicit class argument is lexically enclosing scope */ - func.getScope() = start.getPyClass() and - /* Implicit 'self' is the 0th parameter */ - self.getDefinition().(ParameterDefinition).getDefiningNode() = func.getArg(0).asName().getAFlowNode() - ) - ) - } - - ClassObject startType() { - result = start - } - - ClassObject selfType(PointsToContext ctx) { - PointsTo::ssa_variable_points_to(self, ctx, _, result, _) - } - - predicate instantiation(PointsToContext ctx, ControlFlowNode f) { - PointsTo::points_to(this.(CallNode).getArg(0), ctx, start, _, _) and f = this - } - - EssaVariable getSelf() { - result = self - } -} - -class SuperBoundMethod extends Object { - - override string toString() { - result = "super()." + name - } - - SuperCall superObject; - string name; - ClassObject startType; - - cached - SuperBoundMethod() { - exists(ControlFlowNode object | - this.(AttrNode).getObject(name) = object | - PointsTo::points_to(object, _, superObject, _, _) and - startType = superObject.startType() - ) - } - - FunctionObject getFunction(PointsToContext ctx) { - exists(ClassList mro | - mro = PointsTo::Types::get_mro(superObject.selfType(ctx)) | - result = mro.startingAt(startType).getTail().lookup(name) - ) - } - - predicate instantiation(PointsToContext ctx, ControlFlowNode f) { - PointsTo::points_to(this.(AttrNode).getObject(name), ctx, superObject, _, _) and f = this - } - - EssaVariable getSelf() { - result = superObject.getSelf() - } - -} - diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index 4c1aaf0f3554..39ccc2619fa1 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -3,7 +3,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.Filters -private import semmle.python.pointsto.PointsToContext2 +private import semmle.python.pointsto.PointsToContext private import semmle.python.pointsto.MRO2 private import semmle.python.types.Builtins @@ -31,7 +31,7 @@ library class CfgOrigin extends @py_object { pragma[inline] CfgOrigin fix(ControlFlowNode here) { - if this = unknownValue() then + if this = Builtin::unknown() then result = here else result = this @@ -82,26 +82,26 @@ module CfgOrigin { } CfgOrigin unknown() { - result = unknownValue() + result = Builtin::unknown() } CfgOrigin fromModule(ModuleObjectInternal mod) { - mod.isBuiltin() and result = unknownValue() + mod.isBuiltin() and result = unknown() or result = mod.getSourceModule().getEntryNode() } } -module PointsTo2 { +cached module PointsTo2 { /** INTERNAL -- Use `f.refersTo(value, origin)` instead. */ - predicate points_to(ControlFlowNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + cached predicate points_to(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { points_to_candidate(f, context, value, origin) and reachableBlock(f.getBasicBlock(), context) } - predicate points_to_candidate(ControlFlowNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + private predicate points_to_candidate(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { use_points_to(f, context, value, origin) or /* Not necessary, but for backwards compatibility */ @@ -129,12 +129,12 @@ module PointsTo2 { // f.(CustomPointsToFact).pointsTo(context, value, origin) } - cached CallNode get_a_call(ObjectInternal func, PointsToContext2 context) { + cached CallNode get_a_call(ObjectInternal func, PointsToContext context) { points_to(result.getFunction(), context, func, _) } /* Holds if BasicBlock `b` is reachable, given the context `context`. */ - predicate reachableBlock(BasicBlock b, PointsToContext2 context) { + cached predicate reachableBlock(BasicBlock b, PointsToContext context) { context.appliesToScope(b.getScope()) and not exists(ConditionBlock guard | guard.controls(b, _)) or exists(ConditionBlock guard | @@ -154,7 +154,7 @@ module PointsTo2 { } pragma [noopt] - private predicate allowsFlow(ConditionBlock guard, BasicBlock b, PointsToContext2 context) { + private predicate allowsFlow(ConditionBlock guard, BasicBlock b, PointsToContext context) { exists(ObjectInternal value, boolean sense, ControlFlowNode test | test = guard.getLastNode() and points_to(test, context, value, _) and @@ -166,7 +166,7 @@ module PointsTo2 { /* Holds if the edge `pred` -> `succ` is reachable, given the context `context`. */ pragma [noopt] - predicate controlledReachableEdge(BasicBlock pred, BasicBlock succ, PointsToContext2 context) { + private predicate controlledReachableEdge(BasicBlock pred, BasicBlock succ, PointsToContext context) { exists(ConditionBlock guard, ObjectInternal value, boolean sense, ControlFlowNode test | test = guard.getLastNode() and points_to(test, context, value, _) and @@ -177,7 +177,7 @@ module PointsTo2 { /** Gets an object pointed to by a use (of a variable). */ pragma [noinline] - private predicate use_points_to(NameNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + private predicate use_points_to(NameNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(CfgOrigin origin_or_obj | value != ObjectInternal::undefined() and use_points_to_maybe_origin(f, context, value, origin_or_obj) | @@ -187,12 +187,12 @@ module PointsTo2 { /** Gets an object pointed to by the definition of an ESSA variable. */ pragma [noinline] - private predicate def_points_to(DefinitionNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + private predicate def_points_to(DefinitionNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { points_to(f.getValue(), context, value, origin) } pragma [noinline] - private predicate use_points_to_maybe_origin(NameNode f, PointsToContext2 context, ObjectInternal value, CfgOrigin origin_or_obj) { + private predicate use_points_to_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { ssa_variable_points_to(fast_local_variable(f), context, value, origin_or_obj) or name_lookup_points_to_maybe_origin(f, context, value, origin_or_obj) @@ -203,12 +203,12 @@ module PointsTo2 { /** Holds if `var` refers to `(value, origin)` given the context `context`. */ pragma [noinline] - predicate ssa_variable_points_to(EssaVariable var, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + cached predicate ssa_variable_points_to(EssaVariable var, PointsToContext context, ObjectInternal value, CfgOrigin origin) { ssa_definition_points_to(var.getDefinition(), context, value, origin) } pragma [noinline] - private predicate name_lookup_points_to_maybe_origin(NameNode f, PointsToContext2 context, ObjectInternal value, CfgOrigin origin_or_obj) { + private predicate name_lookup_points_to_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { exists(EssaVariable var | var = name_local_variable(f) | ssa_variable_points_to(var, context, value, origin_or_obj) ) @@ -218,12 +218,12 @@ module PointsTo2 { } pragma [noinline] - private predicate local_variable_undefined(NameNode f, PointsToContext2 context) { + private predicate local_variable_undefined(NameNode f, PointsToContext context) { ssa_variable_points_to(name_local_variable(f), context, ObjectInternal::undefined(), _) } pragma [noinline] - private predicate global_lookup_points_to_maybe_origin(NameNode f, PointsToContext2 context, ObjectInternal value, CfgOrigin origin_or_obj) { + private predicate global_lookup_points_to_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { ssa_variable_points_to(global_variable(f), context, value, origin_or_obj) or exists(ControlFlowNode origin | @@ -260,7 +260,7 @@ module PointsTo2 { /** Holds if `f` is an attribute `x.attr` and points to `(value, cls, origin)`. */ pragma [noinline] - private predicate attribute_load_points_to(AttrNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + private predicate attribute_load_points_to(AttrNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(ObjectInternal object, string name, CfgOrigin orig | points_to(f.getObject(name), context, object, _) | object.attribute(name, value, orig) and @@ -280,7 +280,7 @@ module PointsTo2 { } /** Holds if the ESSA definition `def` refers to `(value, origin)` given the context `context`. */ - predicate ssa_definition_points_to(EssaDefinition def, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + private predicate ssa_definition_points_to(EssaDefinition def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { ssa_phi_points_to(def, context, value, origin) or exists(ControlFlowNode orig | @@ -294,13 +294,13 @@ module PointsTo2 { } pragma [noinline] - private predicate ssa_node_definition_points_to(EssaNodeDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + private predicate ssa_node_definition_points_to(EssaNodeDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { reachableBlock(def.getDefiningNode().getBasicBlock(), _) and ssa_node_definition_points_to_unpruned(def, context, value, origin) } pragma [nomagic] - private predicate ssa_node_definition_points_to_unpruned(EssaNodeDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + private predicate ssa_node_definition_points_to_unpruned(EssaNodeDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { InterProceduralPointsTo::parameter_points_to(def, context, value, origin) or assignment_points_to(def, context, value, origin) @@ -323,7 +323,7 @@ module PointsTo2 { } pragma [noinline] - private predicate ssa_node_refinement_points_to(EssaNodeRefinement def, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + private predicate ssa_node_refinement_points_to(EssaNodeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { //method_callsite_points_to(def, context, value, origin) //or InterModulePointsTo::import_star_points_to(def, context, value, origin) @@ -332,15 +332,20 @@ module PointsTo2 { //or InterProceduralPointsTo::callsite_points_to(def, context, value, origin) or - //argument_points_to(def, context, value, origin) + argument_points_to(def, context, value, origin) //or //attribute_delete_points_to(def, context, value, origin) - //or + or uni_edged_phi_points_to(def, context, value, origin) } + /** Ignore the effects of calls on their arguments. PointsTo is an approximation, but attempting to improve accuracy would be very expensive for very little gain. */ + private predicate argument_points_to(ArgumentRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + ssa_variable_points_to(def.getInput(), context, value, origin) + } + /** Holds if ESSA edge refinement, `def`, refers to `(value, cls, origin)`. */ - predicate ssa_filter_definition_points_to(PyEdgeRefinement def, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + private predicate ssa_filter_definition_points_to(PyEdgeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(ControlFlowNode test, ControlFlowNode use | refinement_test(test, use, Conditionals::branchEvaluatesTo(test, use, context, value, origin.toCfgNode()), def) ) @@ -348,7 +353,7 @@ module PointsTo2 { /** Holds if ESSA definition, `uniphi`, refers to `(value, origin)`. */ pragma [noinline] - predicate uni_edged_phi_points_to(SingleSuccessorGuard uniphi, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + private predicate uni_edged_phi_points_to(SingleSuccessorGuard uniphi, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(ControlFlowNode test, ControlFlowNode use | /* Because calls such as `len` may create a new variable, we need to go via the source variable * That is perfectly safe as we are only dealing with calls that do not mutate their arguments. @@ -361,19 +366,19 @@ module PointsTo2 { /** Points-to for normal assignments `def = ...`. */ pragma [noinline] - private predicate assignment_points_to(AssignmentDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + private predicate assignment_points_to(AssignmentDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { points_to(def.getValue(), context, value, origin) } /** Points-to for deletion: `del name`. */ pragma [noinline] - private predicate delete_points_to(DeletionDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + private predicate delete_points_to(DeletionDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { value = ObjectInternal::undefined() and origin = def.getDefiningNode() and context.appliesToScope(def.getScope()) } /** Implicit "definition" of `__name__` at the start of a module. */ pragma [noinline] - private predicate module_name_points_to(ScopeEntryDefinition def, PointsToContext2 context, StringObjectInternal value, ControlFlowNode origin) { + private predicate module_name_points_to(ScopeEntryDefinition def, PointsToContext context, StringObjectInternal value, ControlFlowNode origin) { def.getVariable().getName() = "__name__" and exists(Module m | m = def.getScope() @@ -397,7 +402,7 @@ module PointsTo2 { /** Holds if the phi-function `phi` refers to `(value, origin)` given the context `context`. */ pragma [nomagic] - private predicate ssa_phi_points_to(PhiFunction phi, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + private predicate ssa_phi_points_to(PhiFunction phi, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(EssaVariable input, BasicBlock pred | input = phi.getInput(pred) and ssa_variable_points_to(input, context, value, origin) @@ -412,9 +417,9 @@ module PointsTo2 { /** Points-to for implicit variable declarations at scope-entry. */ pragma [noinline] - private predicate scope_entry_points_to(ScopeEntryDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + private predicate scope_entry_points_to(ScopeEntryDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { /* Transfer from another scope */ - exists(EssaVariable var, PointsToContext2 outer, CfgOrigin orig | + exists(EssaVariable var, PointsToContext outer, CfgOrigin orig | InterProceduralPointsTo::scope_entry_value_transfer(var, outer, def, context) and ssa_variable_points_to(var, outer, value, orig) and origin = orig.asCfgNodeOrHere(def.getDefiningNode()) @@ -441,7 +446,7 @@ module PointsTo2 { ) } - private predicate subscript_points_to(SubscriptNode sub, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + private predicate subscript_points_to(SubscriptNode sub, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { points_to(sub.getValue(), context, ObjectInternal::unknown(), _) and value = ObjectInternal::unknown() and origin = sub } @@ -449,34 +454,21 @@ module PointsTo2 { /** Track bitwise expressions so we can handle integer flags and enums. * Tracking too many binary expressions is likely to kill performance. */ - private predicate binary_expr_points_to(BinaryExprNode b, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + private predicate binary_expr_points_to(BinaryExprNode b, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { // TO DO... // Track some integer values through `|` and the types of some objects none() } pragma [noinline] - private predicate compare_expr_points_to(CompareNode cmp, PointsToContext2 context, ObjectInternal value) { + private predicate compare_expr_points_to(CompareNode cmp, PointsToContext context, ObjectInternal value) { value = ObjectInternal::bool(Conditionals::comparisonEvaluatesTo(cmp, _, context, _, _)) // or // value = version_tuple_compare(cmp, context) } - /** Helper for comparisons. */ - predicate inequality(CompareNode cmp, ControlFlowNode lesser, ControlFlowNode greater, boolean strict) { - exists(Cmpop op | - cmp.operands(lesser, op, greater) and op.getSymbol() = "<" and strict = true - or - cmp.operands(lesser, op, greater) and op.getSymbol() = "<=" and strict = false - or - cmp.operands(greater, op, lesser) and op.getSymbol() = ">" and strict = true - or - cmp.operands(greater, op, lesser) and op.getSymbol() = ">=" and strict = false - ) - } - pragma [noinline] - private predicate unary_points_to(UnaryExprNode f, PointsToContext2 context, ObjectInternal value) { + private predicate unary_points_to(UnaryExprNode f, PointsToContext context, ObjectInternal value) { exists(Unaryop op, ObjectInternal operand | op = f.getNode().getOp() and points_to(f.getOperand(), context, operand, _) @@ -494,7 +486,7 @@ module PointsTo2 { module InterModulePointsTo { pragma [noinline] - predicate import_points_to(ControlFlowNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + predicate import_points_to(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(string name, ImportExpr i | i.getAFlowNode() = f and i.getImportedModuleName() = name and module_imported_as(value, name) and @@ -503,14 +495,14 @@ module InterModulePointsTo { ) } - predicate from_import_points_to(ImportMemberNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + predicate from_import_points_to(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { from_self_import_points_to(f, context, value, origin) or from_other_import_points_to(f, context, value, origin) } pragma [noinline] - predicate from_self_import_points_to(ImportMemberNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + predicate from_self_import_points_to(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(EssaVariable var, CfgOrigin orig | var = ssa_variable_for_module_attribute(f, context) and PointsTo2::ssa_variable_points_to(var, context, value, orig) and @@ -519,7 +511,7 @@ module InterModulePointsTo { } pragma [noinline] - predicate from_other_import_points_to(ImportMemberNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + predicate from_other_import_points_to(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(string name, ModuleObjectInternal mod, CfgOrigin orig | from_import_imports(f, context, mod, name) and (mod.getSourceModule() != f.getEnclosingModule() or mod.isBuiltin()) and @@ -535,12 +527,12 @@ module InterModulePointsTo { ) } - private predicate from_import_imports(ImportMemberNode f, PointsToContext2 context, ModuleObjectInternal mod, string name) { + private predicate from_import_imports(ImportMemberNode f, PointsToContext context, ModuleObjectInternal mod, string name) { PointsTo2::points_to(f.getModule(name), context, mod, _) } pragma [noinline] - private EssaVariable ssa_variable_for_module_attribute(ImportMemberNode f, PointsToContext2 context) { + private EssaVariable ssa_variable_for_module_attribute(ImportMemberNode f, PointsToContext context) { exists(string name, ModuleObjectInternal mod, Module m | mod.getSourceModule() = m and m = f.getEnclosingModule() and m = result.getScope() and PointsTo2::points_to(f.getModule(name), context, mod, _) and @@ -549,7 +541,7 @@ module InterModulePointsTo { } /* Holds if `import name` will import the module `m`. */ - private predicate module_imported_as(ModuleObjectInternal m, string name) { + predicate module_imported_as(ModuleObjectInternal m, string name) { /* Normal imports */ m.getName() = name or @@ -572,7 +564,7 @@ module InterModulePointsTo { * PointsTo isn't exactly how the interpreter works, but is the best approximation we can manage statically. */ pragma [noinline] - predicate implicit_submodule_points_to(ImplicitSubModuleDefinition def, PointsToContext2 context, ModuleObjectInternal value, ControlFlowNode origin) { + predicate implicit_submodule_points_to(ImplicitSubModuleDefinition def, PointsToContext context, ModuleObjectInternal value, ControlFlowNode origin) { exists(PackageObjectInternal package | package.getSourceModule() = def.getDefiningNode().getScope() | value = package.submodule(def.getSourceVariable().getName()) and @@ -582,7 +574,7 @@ module InterModulePointsTo { } /** Points-to for `from ... import *`. */ - predicate import_star_points_to(ImportStarRefinement def, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + predicate import_star_points_to(ImportStarRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(CfgOrigin orig | origin = orig.fix(def.getDefiningNode()) | @@ -604,7 +596,7 @@ module InterModulePointsTo { /** Holds if `def` is technically a definition of `var`, but the `from ... import *` does not in fact define `var`. */ - cached predicate variable_not_redefined_by_import_star(EssaVariable var, PointsToContext2 context, ImportStarRefinement def) { + cached predicate variable_not_redefined_by_import_star(EssaVariable var, PointsToContext context, ImportStarRefinement def) { var = def.getInput() and exists(ModuleObjectInternal mod | PointsTo2::points_to(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) | @@ -678,12 +670,12 @@ module InterModulePointsTo { module InterProceduralPointsTo { pragma [noinline] - predicate call_points_to(CallNode f, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + predicate call_points_to(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(ObjectInternal func, CfgOrigin resultOrigin | PointsTo2::points_to(f.getFunction(), context, func, _) and origin = resultOrigin.fix(f) | - exists(PointsToContext2 callee | + exists(PointsToContext callee | callee.fromCall(f, context) and func.callResult(callee, value, resultOrigin) ) @@ -694,7 +686,7 @@ module InterProceduralPointsTo { /** Points-to for parameter. `def foo(param): ...`. */ pragma [noinline] - predicate parameter_points_to(ParameterDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { + predicate parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { positional_parameter_points_to(def, context, value, origin) or named_parameter_points_to(def, context, value, origin) @@ -706,8 +698,8 @@ module InterProceduralPointsTo { /** Helper for `parameter_points_to` */ pragma [noinline] - private predicate positional_parameter_points_to(ParameterDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { - exists(PointsToContext2 caller, ControlFlowNode arg | + private predicate positional_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + exists(PointsToContext caller, ControlFlowNode arg | PointsTo2::points_to(arg, caller, value, origin) and callsite_argument_transfer(arg, caller, def, context) ) @@ -719,8 +711,8 @@ module InterProceduralPointsTo { /** Helper for `parameter_points_to` */ pragma [noinline] - private predicate named_parameter_points_to(ParameterDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { - exists(CallNode call, PointsToContext2 caller, PythonFunctionObjectInternal func, string name | + private predicate named_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + exists(CallNode call, PointsToContext caller, PythonFunctionObjectInternal func, string name | context.fromCall(call, func, caller) and def.getParameter() = func.getScope().getArgByName(name) and PointsTo2::points_to(call.getArgByName(name), caller, value, origin) @@ -728,17 +720,17 @@ module InterProceduralPointsTo { } /** Helper for parameter_points_to */ - private predicate default_parameter_points_to(ParameterDefinition def, PointsToContext2 context, ObjectInternal value, ControlFlowNode origin) { - exists(PointsToContext2 imp | imp.isImport() | PointsTo2::points_to(def.getDefault(), imp, value, origin)) and + private predicate default_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + exists(PointsToContext imp | imp.isImport() | PointsTo2::points_to(def.getDefault(), imp, value, origin)) and context_for_default_value(def, context) } /** Helper for default_parameter_points_to */ pragma [noinline] - private predicate context_for_default_value(ParameterDefinition def, PointsToContext2 context) { + private predicate context_for_default_value(ParameterDefinition def, PointsToContext context) { context.isRuntime() or - exists(PointsToContext2 caller, CallNode call, PythonFunctionObjectInternal func, int n | + exists(PointsToContext caller, CallNode call, PythonFunctionObjectInternal func, int n | context.fromCall(call, func, caller) and func.getScope().getArg(n) = def.getParameter() and not exists(call.getArg(n)) and @@ -749,7 +741,7 @@ module InterProceduralPointsTo { } /** Holds if the `(argument, caller)` pair matches up with `(param, callee)` pair across call. */ - cached predicate callsite_argument_transfer(ControlFlowNode argument, PointsToContext2 caller, ParameterDefinition param, PointsToContext2 callee) { + cached predicate callsite_argument_transfer(ControlFlowNode argument, PointsToContext caller, ParameterDefinition param, PointsToContext callee) { exists(CallNode call, Function func, int n, int offset | callsite_calls_function(call, caller, func, callee, offset) and argument = call.getArg(n) and @@ -757,7 +749,7 @@ module InterProceduralPointsTo { ) } - cached predicate callsite_calls_function(CallNode call, PointsToContext2 caller, Function scope, PointsToContext2 callee, int parameter_offset) { + cached predicate callsite_calls_function(CallNode call, PointsToContext caller, Function scope, PointsToContext callee, int parameter_offset) { callee.fromCall(call, caller) and exists(ObjectInternal func | PointsTo2::points_to(call.getFunction(), caller, func, _) and @@ -766,7 +758,7 @@ module InterProceduralPointsTo { } /** Model the transfer of values at scope-entry points. Transfer from `(pred_var, pred_context)` to `(succ_def, succ_context)`. */ - cached predicate scope_entry_value_transfer(EssaVariable pred_var, PointsToContext2 pred_context, ScopeEntryDefinition succ_def, PointsToContext2 succ_context) { + cached predicate scope_entry_value_transfer(EssaVariable pred_var, PointsToContext pred_context, ScopeEntryDefinition succ_def, PointsToContext succ_context) { scope_entry_value_transfer_from_earlier(pred_var, pred_context, succ_def, succ_context) or callsite_entry_value_transfer(pred_var, pred_context, succ_def, succ_context) @@ -778,7 +770,7 @@ module InterProceduralPointsTo { /** Helper for `scope_entry_value_transfer`. Transfer of values from a temporally earlier scope to later scope. * Earlier and later scopes are, for example, a module and functions in that module, or an __init__ method and another method. */ pragma [noinline] - private predicate scope_entry_value_transfer_from_earlier(EssaVariable pred_var, PointsToContext2 pred_context, ScopeEntryDefinition succ_def, PointsToContext2 succ_context) { + private predicate scope_entry_value_transfer_from_earlier(EssaVariable pred_var, PointsToContext pred_context, ScopeEntryDefinition succ_def, PointsToContext succ_context) { exists(Scope pred_scope, Scope succ_scope | BaseFlow::scope_entry_value_transfer_from_earlier(pred_var, pred_scope, succ_def, succ_scope) and succ_context.appliesToScope(succ_scope) @@ -810,7 +802,7 @@ module InterProceduralPointsTo { /** Helper for `scope_entry_value_transfer`. * Transfer of values from the callsite to the callee, for enclosing variables, but not arguments/parameters. */ pragma [noinline] - private predicate callsite_entry_value_transfer(EssaVariable caller_var, PointsToContext2 caller, ScopeEntryDefinition entry_def, PointsToContext2 callee) { + private predicate callsite_entry_value_transfer(EssaVariable caller_var, PointsToContext caller, ScopeEntryDefinition entry_def, PointsToContext callee) { entry_def.getSourceVariable() = caller_var.getSourceVariable() and callsite_calls_function(caller_var.getAUse(), caller, entry_def.getScope(), callee, _) } @@ -829,12 +821,12 @@ module InterProceduralPointsTo { * Where var may be redefined in call to `foo` if `var` escapes (is global or non-local). */ pragma [noinline] - predicate callsite_points_to(CallsiteRefinement def, PointsToContext2 context, ObjectInternal value, CfgOrigin origin) { + predicate callsite_points_to(CallsiteRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(SsaSourceVariable srcvar | srcvar = def.getSourceVariable() | if srcvar instanceof EscapingAssignmentGlobalVariable then ( /* If global variable can be reassigned, we need to track it through calls */ - exists(EssaVariable var, Function func, PointsToContext2 callee | + exists(EssaVariable var, Function func, PointsToContext callee | callsite_calls_function(def.getCall(), context, func, callee, _) and var_at_exit(srcvar, func, var) and PointsTo2::ssa_variable_points_to(var, callee, value, origin) @@ -869,11 +861,11 @@ private predicate potential_builtin_points_to(NameNode f, ObjectInternal value, ( value = ObjectInternal::builtin(f.getId()) or - not exists(Object::builtin(f.getId())) and value = ObjectInternal::unknown() + not exists(Builtin::builtin(f.getId())) and value = ObjectInternal::unknown() ) } -module Conditionals { +private module Conditionals { /** Holds if `expr` is the operand of a unary `not` expression. */ private ControlFlowNode not_operand(ControlFlowNode expr) { @@ -881,7 +873,7 @@ module Conditionals { result = expr.(UnaryExprNode).getOperand() } - boolean branchEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { + boolean branchEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { contains_interesting_expression_within_test(expr, use) and PointsTo2::points_to(use, context, val, origin) and expr = use and @@ -892,7 +884,7 @@ module Conditionals { result = branchEvaluatesTo(not_operand(expr), use, context, val, origin).booleanNot() } - boolean comparisonEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { + boolean comparisonEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { result = equalityEvaluatesTo(expr, use, context, val, origin) or result = inequalityEvaluatesTo(expr, use, context, val, origin) @@ -907,7 +899,7 @@ module Conditionals { } pragma [noinline] - private boolean equalityEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { + private boolean equalityEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { exists(ControlFlowNode r, boolean sense | equality_test(expr, use, sense, r) and exists(ObjectInternal other | @@ -928,13 +920,13 @@ module Conditionals { } pragma [noinline] - private boolean inequalityEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext2 context, ObjectInternal val, ControlFlowNode origin) { + private boolean inequalityEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { exists(ControlFlowNode r, boolean sense | exists(boolean strict, ObjectInternal other | ( - PointsTo2::inequality(expr, use, r, strict) and sense = true + inequality(expr, use, r, strict) and sense = true or - PointsTo2::inequality(expr, r, use, strict) and sense = false + inequality(expr, r, use, strict) and sense = false ) and PointsTo2::points_to(use, context, val, origin) and PointsTo2::points_to(r, context, other, _) @@ -959,11 +951,24 @@ module Conditionals { ) } + /** Helper for comparisons. */ + private predicate inequality(CompareNode cmp, ControlFlowNode lesser, ControlFlowNode greater, boolean strict) { + exists(Cmpop op | + cmp.operands(lesser, op, greater) and op.getSymbol() = "<" and strict = true + or + cmp.operands(lesser, op, greater) and op.getSymbol() = "<=" and strict = false + or + cmp.operands(greater, op, lesser) and op.getSymbol() = ">" and strict = true + or + cmp.operands(greater, op, lesser) and op.getSymbol() = ">=" and strict = false + ) + } + } -module Types { +cached module Types { - int base_count(ClassObjectInternal cls) { + cached int base_count(ClassObjectInternal cls) { cls = ObjectInternal::builtin("object") and result = 0 or exists(cls.getBuiltin()) and cls != ObjectInternal::builtin("object") and result = 1 @@ -978,7 +983,7 @@ module Types { ) } - ClassObjectInternal getBase(ClassObjectInternal cls, int n) { + cached ClassObjectInternal getBase(ClassObjectInternal cls, int n) { result.getBuiltin() = cls.getBuiltin().getBaseClass() and n = 0 or exists(Class pycls | @@ -993,17 +998,17 @@ module Types { result = ObjectInternal::builtin("object") } - predicate isOldStyle(ClassObjectInternal cls) { + cached predicate isOldStyle(ClassObjectInternal cls) { //To do... none() } - predicate isNewStyle(ClassObjectInternal cls) { + cached predicate isNewStyle(ClassObjectInternal cls) { //To do... any() } - ClassList getMro(ClassObjectInternal cls) { + cached ClassList getMro(ClassObjectInternal cls) { isNewStyle(cls) and result = Mro::newStyleMro(cls) or @@ -1011,7 +1016,7 @@ module Types { none() } - predicate declaredAttribute(ClassObjectInternal cls, string name, ObjectInternal value, CfgOrigin origin) { + cached predicate declaredAttribute(ClassObjectInternal cls, string name, ObjectInternal value, CfgOrigin origin) { value = ObjectInternal::fromBuiltin(cls.getBuiltin().getMember(name)) and origin = CfgOrigin::unknown() or value != ObjectInternal::undefined() and @@ -1022,7 +1027,7 @@ module Types { ) } - ClassObjectInternal getMetaClass(PythonClassObjectInternal cls) { + cached ClassObjectInternal getMetaClass(PythonClassObjectInternal cls) { result = declaredMetaClass(cls) or hasDeclaredMetaclass(cls) = false and result = getInheritedMetaclass(cls) @@ -1048,7 +1053,7 @@ module Types { private boolean has_six_add_metaclass(PythonClassObjectInternal cls) { // TO DO... - none() + result = false } private boolean has_metaclass_var_metaclass(PythonClassObjectInternal cls) { @@ -1120,7 +1125,7 @@ module Types { ) or exists(ClassObjectInternal meta1, ClassObjectInternal meta2 | - meta1 = getMetaClass(getBase(cls, n)) and + meta1 = getBase(cls, n).getClass() and meta2 = getInheritedMetaclass(cls, n+1) | /* Choose sub-class */ @@ -1141,6 +1146,32 @@ module Types { result = improperSuperType(getBase(cls, _)) } + /* Holds if type inference failed to compute the full class hierarchy for this class for the reason given. */ + cached predicate failedInference(ClassObjectInternal cls, string reason) { + strictcount(cls.(PythonClassObjectInternal).getScope().getADecorator()) > 1 and reason = "Multiple decorators" + or + exists(cls.(PythonClassObjectInternal).getScope().getADecorator()) and not six_add_metaclass(_, cls, _) and reason = "Decorator not understood" + or + exists(int i | + exists(cls.(PythonClassObjectInternal).getScope().getBase(i)) and reason = "Missing base " + i + | + not exists(getBase(cls, i)) + ) + or + exists(cls.(PythonClassObjectInternal).getScope().getMetaClass()) and not exists(cls.getClass()) and reason = "Failed to infer metaclass" + or + exists(int i | failedInference(getBase(cls, i), _) and reason = "Failed inference for base class at position " + i) + or + exists(int i, ObjectInternal base1, ObjectInternal base2 | + base1 = getBase(cls, i) and + base2 = getBase(cls, i) and + base1 != base2 and + reason = "Multiple bases at position " + i + ) + or + exists(int i, int j | getBase(cls, i) = getBase(cls, j) and i != j and reason = "Duplicate bases classes") + } + } diff --git a/python/ql/src/semmle/python/pointsto/PointsToContext.qll b/python/ql/src/semmle/python/pointsto/PointsToContext.qll old mode 100755 new mode 100644 index abbf5117e666..3558ef4a177a --- a/python/ql/src/semmle/python/pointsto/PointsToContext.qll +++ b/python/ql/src/semmle/python/pointsto/PointsToContext.qll @@ -1,6 +1,6 @@ import python -private import semmle.python.pointsto.PointsTo - +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.objects.ObjectInternal /* * A note on 'cost'. Cost doesn't represent the cost to compute, * but (a vague estimate of) the cost to compute per value gained. @@ -8,7 +8,7 @@ private import semmle.python.pointsto.PointsTo */ private int given_cost() { - exists(string depth | + exists(string depth | py_flags_versioned("context.cost", depth, _) and result = depth.toInt() ) @@ -148,8 +148,8 @@ class PointsToContext extends TPointsToContext { } /** Holds if `call` is the call-site from which this context was entered and `caller` is the caller's context. */ - predicate fromCall(CallNode call, FunctionObject callee, PointsToContext caller) { - call = PointsTo::get_a_call(callee, caller) and + predicate fromCall(CallNode call, PythonFunctionObjectInternal callee, PointsToContext caller) { + call = PointsTo2::get_a_call(callee, caller) and this = TCallContext(call, caller, _) } @@ -169,16 +169,13 @@ class PointsToContext extends TPointsToContext { this = TRuntimeContext() and executes_in_runtime_context(s) or /* Called functions, regardless of their name */ - exists(FunctionObject func, ControlFlowNode call, TPointsToContext outerContext | - call = PointsTo::get_a_call(func, outerContext) and + exists(PythonFunctionObjectInternal func, ControlFlowNode call, TPointsToContext outerContext | + call = PointsTo2::get_a_call(func, outerContext) and this = TCallContext(call, outerContext, _) and - s = func.getFunction() + s = func.getScope() ) or - exists(FunctionObject func | - PointsTo::Flow::callsite_calls_function(_, _, func, this, _) and - s = func.getFunction() - ) + InterProceduralPointsTo::callsite_calls_function(_, _, s, this, _) } /** Holds if this context can apply to the CFG node `n`. */ @@ -225,6 +222,10 @@ class PointsToContext extends TPointsToContext { result = context_cost(this) } + CallNode getCall() { + this = TCallContext(result, _, _) + } + /** Holds if a call would be too expensive to create a new context for */ predicate untrackableCall(CallNode call) { total_cost(call, this) > max_context_cost() @@ -269,8 +270,3 @@ private predicate maybe_main(Module m) { ) } - -/* For backwards compatibility */ -/** DEPRECATED: Use `PointsToContext` instead */ -deprecated class FinalContext = PointsToContext; - diff --git a/python/ql/src/semmle/python/pointsto/PointsToContext2.qll b/python/ql/src/semmle/python/pointsto/PointsToContext2.qll deleted file mode 100644 index 4473ab8f30f0..000000000000 --- a/python/ql/src/semmle/python/pointsto/PointsToContext2.qll +++ /dev/null @@ -1,272 +0,0 @@ -import python -private import semmle.python.pointsto.PointsTo2 -private import semmle.python.objects.ObjectInternal -/* - * A note on 'cost'. Cost doesn't represent the cost to compute, - * but (a vague estimate of) the cost to compute per value gained. - * This is constantly evolving, so see the various cost functions below for more details. - */ - -private int given_cost() { - exists(string depth | - py_flags_versioned("context.cost", depth, _) and - result = depth.toInt() - ) -} - -private int max_context_cost() { - not py_flags_versioned("context.cost", _, _) and result = 7 - or - result = max(int cost | cost = given_cost() | cost) -} - -private int syntactic_call_count(Scope s) { - exists(Function f | - f = s and f.getName() != "__init__" | - result = count(CallNode call | - call.getFunction().(NameNode).getId() = f.getName() - or - call.getFunction().(AttrNode).getName() = f.getName() - ) - ) - or - s.getName() = "__init__" and result = 1 - or - not s instanceof Function and result = 0 -} - -private int incoming_call_cost(Scope s) { - /* Syntactic call count will often be a considerable overestimate - * of the actual number of calls, so we use the square root. - * Cost = log(sqrt(call-count)) - */ - result = ((syntactic_call_count(s)+1).log(2)*0.5).floor() -} - -private int context_cost(TPointsToContext ctx) { - ctx = TMainContext() and result = 0 - or - ctx = TRuntimeContext() and result = 0 - or - ctx = TImportContext() and result = 0 - or - ctx = TCallContext(_, _, result) -} - -private int call_cost(CallNode call) { - if call.getScope().inSource() then - result = 2 - else - result = 3 -} - -private int outgoing_calls(Scope s) { - result = strictcount(CallNode call | call.getScope() = s) -} - -predicate super_method_call(CallNode call) { - call.getFunction().(AttrNode).getObject().(CallNode).getFunction().(NameNode).getId() = "super" -} - -private int outgoing_call_cost(CallNode c) { - /* Cost = log(outgoing-call-count) */ - result = outgoing_calls(c.getScope()).log(2).floor() -} - -/** Cost of contexts for a call, the more callers the - * callee of call has the more expensive it is to add contexts for it. - * This seems to be an effective heuristics for preventing an explosion - * in the number of contexts while retaining good results. - */ -private int splay_cost(CallNode c) { - if super_method_call(c) then - result = 0 - else - result = outgoing_call_cost(c) + incoming_call_cost(c.getScope()) -} - -private predicate call_to_init_or_del(CallNode call) { - exists(string mname | - mname = "__init__" or mname = "__del__" | - mname = call.getFunction().(AttrNode).getName() - ) -} - -/** Total cost estimate */ -private int total_call_cost(CallNode call) { - /* We want to always follow __init__ and __del__ calls as they tell us about object construction, - * but we need to be aware of cycles, so they must have a non-zero cost. - */ - if call_to_init_or_del(call) then - result = 1 - else - result = call_cost(call) + splay_cost(call) -} - -private int total_cost(CallNode call, PointsToContext2 ctx) { - ctx.appliesTo(call) and - result = total_call_cost(call) + context_cost(ctx) -} - -private cached newtype TPointsToContext = - TMainContext() - or - TRuntimeContext() - or - TImportContext() - or - TCallContext(ControlFlowNode call, PointsToContext2 outerContext, int cost) { - total_cost(call, outerContext) = cost and - cost <= max_context_cost() - } - -/** Points-to context. Context can be one of: - * * "main": Used for scripts. - * * "import": Use for non-script modules. - * * "default": Use for functions and methods without caller context. - * * All other contexts are call contexts and consist of a pair of call-site and caller context. - */ -class PointsToContext2 extends TPointsToContext { - - cached string toString() { - this = TMainContext() and result = "main" - or - this = TRuntimeContext() and result = "runtime" - or - this = TImportContext() and result = "import" - or - exists(CallNode callsite, PointsToContext2 outerContext | - this = TCallContext(callsite, outerContext, _) and - result = callsite.getLocation() + " from " + outerContext.toString() - ) - } - - /** Holds if `call` is the call-site from which this context was entered and `outer` is the caller's context. */ - predicate fromCall(CallNode call, PointsToContext2 caller) { - caller.appliesTo(call) and - this = TCallContext(call, caller, _) - } - - /** Holds if `call` is the call-site from which this context was entered and `caller` is the caller's context. */ - predicate fromCall(CallNode call, PythonFunctionObjectInternal callee, PointsToContext2 caller) { - call = PointsTo2::get_a_call(callee, caller) and - this = TCallContext(call, caller, _) - } - - /** Gets the caller context for this callee context. */ - PointsToContext2 getOuter() { - this = TCallContext(_, result, _) - } - - /** Holds if this context is relevant to the given scope. */ - predicate appliesToScope(Scope s) { - /* Scripts */ - this = TMainContext() and maybe_main(s) - or - /* Modules and classes evaluated at import */ - s instanceof ImportTimeScope and this = TImportContext() - or - this = TRuntimeContext() and executes_in_runtime_context(s) - or - /* Called functions, regardless of their name */ - exists(PythonFunctionObjectInternal func, ControlFlowNode call, TPointsToContext outerContext | - call = PointsTo2::get_a_call(func, outerContext) and - this = TCallContext(call, outerContext, _) and - s = func.getScope() - ) - or - InterProceduralPointsTo::callsite_calls_function(_, _, s, this, _) - } - - /** Holds if this context can apply to the CFG node `n`. */ - pragma [inline] - predicate appliesTo(ControlFlowNode n) { - this.appliesToScope(n.getScope()) - } - - /** Holds if this context is a call context. */ - predicate isCall() { - this = TCallContext(_, _, _) - } - - /** Holds if this is the "main" context. */ - predicate isMain() { - this = TMainContext() - } - - /** Holds if this is the "import" context. */ - predicate isImport() { - this = TImportContext() - } - - /** Holds if this is the "default" context. */ - predicate isRuntime() { - this = TRuntimeContext() - } - - /** Holds if this context or one of its caller contexts is the default context. */ - predicate fromRuntime() { - this.isRuntime() - or - this.getOuter().fromRuntime() - } - - /** Gets the depth (number of calls) for this context. */ - int getDepth() { - not exists(this.getOuter()) and result = 0 - or - result = this.getOuter().getDepth() + 1 - } - - int getCost() { - result = context_cost(this) - } - - CallNode getCall() { - this = TCallContext(result, _, _) - } - - /** Holds if a call would be too expensive to create a new context for */ - predicate untrackableCall(CallNode call) { - total_cost(call, this) > max_context_cost() - } - - CallNode getRootCall() { - this = TCallContext(result, TImportContext(), _) - or - result = this.getOuter().getRootCall() - } - - /** Gets a version of Python that this context includes */ - pragma [inline] - Version getAVersion() { - /* Currently contexts do not include any version information, but may do in the future */ - result = major_version() - } - -} - -private predicate in_source(Scope s) { - exists(s.getEnclosingModule().getFile().getRelativePath()) -} - -/** Holds if this scope can be executed in the default context. - * All modules and classes executed at import time and - * all "public" functions and methods, including those invoked by the VM. - */ -predicate executes_in_runtime_context(Function f) { - /* "Public" scope, i.e. functions whose name starts not with an underscore, or special methods */ - (f.getName().charAt(0) != "_" or f.isSpecialMethod() or f.isInitMethod()) - and - in_source(f) -} - -private predicate maybe_main(Module m) { - exists(If i, Compare cmp, Name name, StrConst main | - m.getAStmt() = i and i.getTest() = cmp | - cmp.compares(name, any(Eq eq), main) and - name.getId() = "__name__" and - main.getText() = "__main__" - ) -} - diff --git a/python/ql/src/semmle/python/types/ClassObject.qll b/python/ql/src/semmle/python/types/ClassObject.qll index d11c3a2fdb26..faa2da56f611 100644 --- a/python/ql/src/semmle/python/types/ClassObject.qll +++ b/python/ql/src/semmle/python/types/ClassObject.qll @@ -1,7 +1,8 @@ import python -private import semmle.python.pointsto.PointsTo -private import semmle.python.pointsto.Base -private import semmle.python.pointsto.MRO as MRO +private import semmle.python.objects.Classes +private import semmle.python.objects.Instances +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.MRO2 private import semmle.python.types.Builtins @@ -20,38 +21,34 @@ private import semmle.python.types.Builtins */ class ClassObject extends Object { + private ClassObjectInternal theClass() { + result.getOrigin() = this + or + result.getBuiltin() = this + } + ClassObject() { this.getOrigin() instanceof ClassExpr or this.asBuiltin().isClass() } - private predicate isStr() { - this.asBuiltin() = Builtin::special("bytes") and major_version() = 2 - or - this.asBuiltin() = Builtin::special("unicode") and major_version() = 3 - } - /** Gets the short (unqualified) name of this class */ string getName() { - this.isStr() and result = "str" - or - not this.isStr() and result = this.asBuiltin().getName() and not this = theUnknownType() - or - result = this.getPyClass().getName() + result = theClass().getName() } /** Gets the qualified name for this class. * Should return the same name as the `__qualname__` attribute on classes in Python 3. */ string getQualifiedName() { - this.isBuiltin() and result = this.getName() + result = theClass().getBuiltin().getName() or - result = this.getPyClass().getQualifiedName() + result = theClass().(PythonClassObjectInternal).getScope().getQualifiedName() } /** Gets the nth base class of this class */ Object getBaseType(int n) { - result = PointsTo::Types::class_base_type(this, n) + result = Types::getBase(theClass(), n).getSource() } /** Gets a base class of this class */ @@ -61,25 +58,23 @@ class ClassObject extends Object { /** Whether this class has a base class */ predicate hasABase() { - exists(ClassExpr cls | this.getOrigin() = cls | exists(cls.getABase())) - or - exists(this.asBuiltin().getBaseClass()) + exists(Types::getBase(theClass(), _)) } /** Gets a super class of this class (includes transitive super classes) */ ClassObject getASuperType() { - result = PointsTo::Types::get_a_super_type(this) + result = Types::getMro(theClass()).getTail().getAnItem().getSource() } /** Gets a super class of this class (includes transitive super classes) or this class */ ClassObject getAnImproperSuperType() { - result = PointsTo::Types::get_an_improper_super_type(this) + result = Types::getMro(theClass()).getAnItem().getSource() } /** Whether this class is a new style class. A new style class is one that implicitly or explicitly inherits from `object`. */ predicate isNewStyle() { - PointsTo::Types::is_new_style(this) + Types::isNewStyle(theClass()) } /** Whether this class is a legal exception class. @@ -98,52 +93,57 @@ class ClassObject extends Object { /** Returns an attribute declared on this class (not on a super-class) */ Object declaredAttribute(string name) { - PointsTo::Types::class_declared_attribute(this, name, result, _, _) + exists(Value val | + Types::declaredAttribute(theClass(), name, val, _) and + result = val.getSource() + ) } /** Returns an attribute declared on this class (not on a super-class) */ predicate declaresAttribute(string name) { - class_declares_attribute(this, name) + theClass().getClassDeclaration().declaresAttribute(name) } /** Returns an attribute as it would be when looked up at runtime on this class. Will include attributes of super-classes */ Object lookupAttribute(string name) { - result = this.getMro().lookup(name) + exists(Value val | + theClass().attribute(name, val, _) and + result = val.getSource() + ) } - MRO::ClassList getMro() { - PointsTo::Types::is_new_style_bool(this) = true and - result = MRO::new_style_mro(this) - or - result = MRO::old_style_mro(this) + ClassList getMro() { + result = Types::getMro(theClass()) } /** Looks up an attribute by searching this class' MRO starting at `start` */ Object lookupMro(ClassObject start, string name) { - result = this.getMro().startingAt(start).lookup(name) + exists(ClassObjectInternal other, ClassObjectInternal decl, Value val | + other.getSource() = start and + decl = Types::getMro(theClass()).startingAt(other).findDeclaringClass(name) and + Types::declaredAttribute(decl, name, val, _) and + result = val.getSource() + ) } /** Whether the named attribute refers to the object and origin */ predicate attributeRefersTo(string name, Object obj, ControlFlowNode origin) { - exists(CfgOrigin orig | - origin = orig.toCfgNode() and - PointsTo::Types::class_attribute_lookup(this, name, obj, _, orig) - ) + this.attributeRefersTo(name, obj, _, origin) } /** Whether the named attribute refers to the object, class and origin */ predicate attributeRefersTo(string name, Object obj, ClassObject cls, ControlFlowNode origin) { - not obj = unknownValue() and - exists(CfgOrigin orig | - origin = orig.toCfgNode() and - PointsTo::Types::class_attribute_lookup(this, name, obj, cls, orig) + exists(Value val | + theClass().attribute(name, val, origin) and + obj = val.getSource() and + cls = val.getClass().getSource() ) } /** Whether this class has a attribute named `name`, either declared or inherited.*/ predicate hasAttribute(string name) { - PointsTo::Types::class_has_attribute(this, name) + Types::getMro(theClass()).getAnItem().getClassDeclaration().declaresAttribute(name) } /** Whether it is impossible to know all the attributes of this class. Usually because it is @@ -164,7 +164,7 @@ class ClassObject extends Object { /** Gets the metaclass for this class */ ClassObject getMetaClass() { - result = PointsTo::Types::class_get_meta_class(this) + result = theClass().getClass().getSource() and not this.failedInference() } @@ -187,7 +187,7 @@ class ClassObject extends Object { /** Has type inference failed to compute the full class hierarchy for this class for the reason given. */ predicate failedInference(string reason) { - PointsTo::Types::failed_inference(this, reason) + Types::failedInference(theClass(), reason) } /** Has type inference failed to compute the full class hierarchy for this class */ @@ -213,8 +213,8 @@ class ClassObject extends Object { } /** This class is only instantiated at one place in the code */ - private predicate hasStaticallyUniqueInstance() { - strictcount(Object instances | PointsTo::points_to(_, _, instances, this, _)) = 1 + private predicate hasStaticallyUniqueInstance() { + strictcount(SpecificInstanceInternal inst | inst.getClass() = theClass()) = 1 } ImportTimeScope getImportTimeScope() { @@ -231,10 +231,9 @@ class ClassObject extends Object { /** Returns the next class in the MRO of 'this' after 'sup' */ ClassObject nextInMro(ClassObject sup) { - exists(MRO::ClassList mro, int i | - mro = this.getMro() and - sup = mro.getItem(i) and - result = mro.getItem(i+1) + exists(ClassObjectInternal other | + other.getSource() = sup and + result = Types::getMro(theClass()).startingAt(other).getTail().getHead().getSource() ) and not this.failedInference() } @@ -243,7 +242,7 @@ class ClassObject extends Object { * `this` has an index of `1`, the next class in the MRO has an index of `2`, and so on. */ ClassObject getMroItem(int index) { - result = this.getMro().getItem(index) + result = this.getMro().getItem(index).getSource() } /** Holds if this class has duplicate base classes */ diff --git a/python/ql/src/semmle/python/types/Descriptors.qll b/python/ql/src/semmle/python/types/Descriptors.qll index 168597d0eed9..2b183c5445ac 100644 --- a/python/ql/src/semmle/python/types/Descriptors.qll +++ b/python/ql/src/semmle/python/types/Descriptors.qll @@ -1,45 +1,20 @@ import python -private import semmle.python.pointsto.PointsTo +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.objects.ObjectInternal -/** A bound method object, x.f where type(x) has a method f */ -class BoundMethod extends Object { - - BoundMethod() { - bound_method(this, _) - } - - /* Gets the method 'f' in 'x.f' */ - FunctionObject getMethod() { - bound_method(this, result) - } - -} - -private predicate bound_method(AttrNode binding, FunctionObject method) { - binding = method.getAMethodCall().getFunction() -} - -private predicate decorator_call(Object method, ClassObject decorator, FunctionObject func) { - exists(CallNode f | - method = f and - f.getFunction().refersTo(decorator) and - PointsTo::points_to(f.getArg(0), _, func, _, _) - ) -} /** A class method object. Either a decorated function or an explicit call to classmethod(f) */ class ClassMethodObject extends Object { ClassMethodObject() { - PointsTo::class_method(this, _) + any(ClassMethodObjectInternal cm).getOrigin() = this } FunctionObject getFunction() { - PointsTo::class_method(this, result) - } - - CallNode getACall() { - PointsTo::class_method_call(this, _, _, _, result) + exists(ClassMethodObjectInternal cm | + cm.getOrigin() = this and + result = cm.getFunction().getSource() + ) } } @@ -48,11 +23,14 @@ class ClassMethodObject extends Object { class StaticMethodObject extends Object { StaticMethodObject() { - decorator_call(this, theStaticMethodType(), _) + any(StaticMethodObjectInternal sm).getOrigin() = this } FunctionObject getFunction() { - decorator_call(this, theStaticMethodType(), result) + exists(StaticMethodObjectInternal sm | + sm.getOrigin() = this and + result = sm.getFunction().getSource() + ) } } diff --git a/python/ql/src/semmle/python/types/FunctionObject.qll b/python/ql/src/semmle/python/types/FunctionObject.qll index 9c417e99c973..4f1115a3997a 100644 --- a/python/ql/src/semmle/python/types/FunctionObject.qll +++ b/python/ql/src/semmle/python/types/FunctionObject.qll @@ -1,6 +1,7 @@ import python import semmle.python.types.Exceptions -private import semmle.python.pointsto.PointsTo +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.objects.Callables private import semmle.python.libraries.Zope private import semmle.python.pointsto.Base private import semmle.python.types.Builtins @@ -8,6 +9,10 @@ private import semmle.python.types.Builtins /** A function object, whether written in Python or builtin */ abstract class FunctionObject extends Object { + CallableValue theCallable() { + result.getSource() = this + } + predicate isOverridingMethod() { exists(Object f | this.overrides(f)) } @@ -42,42 +47,44 @@ abstract class FunctionObject extends Object { /** Gets a call-site from where this function is called as a function */ CallNode getAFunctionCall() { - PointsTo::function_call(this, _, result) + result.getFunction() instanceof NameNode and + result = this.getACall() } /** Gets a call-site from where this function is called as a method */ CallNode getAMethodCall() { - PointsTo::method_call(this, _, result) + result.getFunction() instanceof AttrNode and + result = this.getACall() } /** Gets a call-site from where this function is called */ ControlFlowNode getACall() { - result = PointsTo::get_a_call(this, _) + result = PointsTo2::get_a_call(theCallable(), _) } /** Gets a call-site from where this function is called, given the `context` */ ControlFlowNode getACall(Context caller_context) { - result = PointsTo::get_a_call(this, caller_context) + result = PointsTo2::get_a_call(theCallable(), caller_context) } /** Gets the `ControlFlowNode` that will be passed as the nth argument to `this` when called at `call`. This predicate will correctly handle `x.y()`, treating `x` as the zeroth argument. */ ControlFlowNode getArgumentForCall(CallNode call, int n) { - result = PointsTo::get_positional_argument_for_call(this, _, call, n) + result = theCallable().getArgumentForCall(call, n) } /** Gets the `ControlFlowNode` that will be passed as the named argument to `this` when called at `call`. This predicate will correctly handle `x.y()`, treating `x` as the self argument. */ ControlFlowNode getNamedArgumentForCall(CallNode call, string name) { - result = PointsTo::get_named_argument_for_call(this, _, call, name) + result = theCallable().getNamedArgumentForCall(call, name) } /** Whether this function never returns. This is an approximation. */ predicate neverReturns() { - PointsTo::function_never_returns(this) + theCallable().neverReturns() } /** Whether this is a "normal" method, that is, it is exists as a class attribute @@ -133,7 +140,7 @@ abstract class FunctionObject extends Object { class PyFunctionObject extends FunctionObject { PyFunctionObject() { - this.getOrigin() instanceof CallableExpr + any(PythonFunctionObjectInternal f).getOrigin() = this } override string toString() { @@ -255,12 +262,7 @@ abstract class BuiltinCallable extends FunctionObject { class BuiltinMethodObject extends BuiltinCallable { BuiltinMethodObject() { - this.asBuiltin().getClass() = theMethodDescriptorType().asBuiltin() - or - this.asBuiltin().getClass() = theBuiltinFunctionType().asBuiltin() and - exists(Builtin cls | cls.isClass() and cls.getMember(_) = this.asBuiltin()) - or - this.asBuiltin().getClass().getName() = "wrapper_descriptor" + any(BuiltinMethodObjectInternal m).getBuiltin() = this } override string getQualifiedName() { @@ -312,8 +314,7 @@ class BuiltinMethodObject extends BuiltinCallable { class BuiltinFunctionObject extends BuiltinCallable { BuiltinFunctionObject() { - this.asBuiltin().getClass() = theBuiltinFunctionType().asBuiltin() and - exists(ModuleObject m | m.asBuiltin().getMember(_) = this.asBuiltin()) + any(BuiltinFunctionObjectInternal f).getBuiltin() = this } override string getName() { diff --git a/python/ql/src/semmle/python/types/ModuleObject.qll b/python/ql/src/semmle/python/types/ModuleObject.qll index 82425c82f95a..90076fce7d97 100644 --- a/python/ql/src/semmle/python/types/ModuleObject.qll +++ b/python/ql/src/semmle/python/types/ModuleObject.qll @@ -1,14 +1,12 @@ import python -private import semmle.python.pointsto.PointsTo -private import semmle.python.pointsto.Base +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.objects.Modules private import semmle.python.types.ModuleKind abstract class ModuleObject extends Object { - ModuleObject () { - exists(Module m | m.getEntryNode() = this) - or - this.asBuiltin().getClass() = theModuleType().asBuiltin() + ModuleValue theModule() { + result.getSource() = this } /** Gets the scope corresponding to this module, if this is a Python module */ @@ -35,8 +33,9 @@ abstract class ModuleObject extends Object { /** Gets the named attribute of this module. Using attributeRefersTo() instead * may provide better results for presentation. * */ - pragma [noinline] - abstract Object getAttribute(string name); + Object getAttribute(string name) { + this.attributeRefersTo(name, result, _) + } /** Gets the named attribute of this module. * Synonym for `getAttribute(name)` */ @@ -45,13 +44,17 @@ abstract class ModuleObject extends Object { result = this.getAttribute(name) } - /** Whether the named attribute of this module "refers-to" value, with a known origin. - */ - abstract predicate attributeRefersTo(string name, Object value, ControlFlowNode origin); - /** Whether the named attribute of this module "refers-to" value, with known class and a known origin. - */ - abstract predicate attributeRefersTo(string name, Object value, ClassObject cls, ControlFlowNode origin); + predicate hasAttribute(string name) { + exists(theModule().attr(name)) + } + + predicate attributeRefersTo(string name, Object obj, ControlFlowNode origin) { + exists(Value val | + theModule().(ModuleObjectInternal).attribute(name, val, origin) and + obj = val.getSource() + ) + } /** Gets the package for this module. */ PackageObject getPackage() { @@ -62,7 +65,7 @@ abstract class ModuleObject extends Object { /** Whether this module "exports" `name`. That is, whether using `import *` on this module will result in `name` being added to the namespace. */ predicate exports(string name) { - PointsTo::module_exports(this, name) + theModule().exports(name) } /** Whether the complete set of names "exported" by this module can be accurately determined */ @@ -78,11 +81,9 @@ abstract class ModuleObject extends Object { /** Whether this module is imported by 'import name'. For example on a linux system, * the module 'posixpath' is imported as 'os.path' or as 'posixpath' */ predicate importedAs(string name) { - PointsTo::module_imported_as(this, name) + InterModulePointsTo::module_imported_as(theModule(), name) } - abstract predicate hasAttribute(string name); - ModuleObject getAnImportedModule() { result.importedAs(this.getModule().getAnImportedModuleName()) } @@ -118,14 +119,6 @@ class BuiltinModuleObject extends ModuleObject { exists(this.asBuiltin().getMember(name)) } - override predicate attributeRefersTo(string name, Object value, ControlFlowNode origin) { - none() - } - - override predicate attributeRefersTo(string name, Object value, ClassObject cls, ControlFlowNode origin) { - none() - } - override predicate exportsComplete() { any() } @@ -157,10 +150,6 @@ class PythonModuleObject extends ModuleObject { result = this.getModule().getFile() } - override Object getAttribute(string name) { - this.attributeRefersTo(name, result, _, _) - } - override predicate exportsComplete() { exists(Module m | m = this.getModule() | @@ -172,29 +161,6 @@ class PythonModuleObject extends ModuleObject { ) } - override predicate hasAttribute(string name) { - PointsTo::module_defines_name(this.getModule(), name) - or - this.attributeRefersTo(name, _, _, _) - or - /* The interpreter always adds the __name__ and __package__ attributes */ - name = "__name__" or name = "__package__" - } - - override predicate attributeRefersTo(string name, Object value, ControlFlowNode origin) { - exists(CfgOrigin orig | - origin = orig.toCfgNode() and - PointsTo::py_module_attributes(this.getModule(), name, value, _, orig) - ) - } - - override predicate attributeRefersTo(string name, Object value, ClassObject cls, ControlFlowNode origin) { - exists(CfgOrigin orig | - origin = orig.toCfgNode() and - PointsTo::py_module_attributes(this.getModule(), name, value, cls, orig) - ) - } - } /** Primarily for internal use. @@ -246,7 +212,10 @@ class PackageObject extends ModuleObject { } override Object getAttribute(string name) { - PointsTo::package_attribute_points_to(this, name, result, _, _) + exists(Value val | + theModule().(PackageObjectInternal).attribute(name, _, _) and + result = val.getSource() + ) } PythonModuleObject getInitModule() { @@ -273,20 +242,6 @@ class PackageObject extends ModuleObject { this.getInitModule().hasAttribute(name) } - override predicate attributeRefersTo(string name, Object value, ControlFlowNode origin) { - exists(CfgOrigin orig | - origin = orig.toCfgNode() and - PointsTo::package_attribute_points_to(this, name, value, _, orig) - ) - } - - override predicate attributeRefersTo(string name, Object value, ClassObject cls, ControlFlowNode origin) { - exists(CfgOrigin orig | - origin = orig.toCfgNode() and - PointsTo::package_attribute_points_to(this, name, value, cls, orig) - ) - } - Location getLocation() { none() } diff --git a/python/ql/src/semmle/python/types/Object.qll b/python/ql/src/semmle/python/types/Object.qll index fd87c4a64d6b..37315eefa627 100644 --- a/python/ql/src/semmle/python/types/Object.qll +++ b/python/ql/src/semmle/python/types/Object.qll @@ -1,5 +1,6 @@ import python -private import semmle.python.pointsto.Base +private import semmle.python.objects.ObjectAPI +private import semmle.python.objects.ObjectInternal private import semmle.python.types.Builtins private cached predicate is_an_object(@py_object obj) { @@ -121,40 +122,25 @@ class Object extends @py_object { ) } + private boolean booleanFromValue() { + exists(ObjectInternal obj | + obj.getSource() = this | + result = obj.booleanValue() + ) + } + /** The Boolean value of this object if it always evaluates to true or false. * For example: * false for None, true for 7 and no result for int(x) */ boolean booleanValue() { - this = theNoneObject() and result = false - or - this = theTrueObject() and result = true - or - this = theFalseObject() and result = false - or - this = TupleObject::empty() and result = false - or - exists(Tuple t | t = this.getOrigin() | - exists(t.getAnElt()) and result = true - or - not exists(t.getAnElt()) and result = false - ) - or - exists(Unicode s | s.getLiteralObject() = this | - s.getS() = "" and result = false - or - s.getS() != "" and result = true - ) - or - exists(Bytes s | s.getLiteralObject() = this | - s.getS() = "" and result = false - or - s.getS() != "" and result = true - ) + result = booleanFromValue() + and not booleanFromValue() = result.booleanNot() } final predicate maybe() { - not exists(this.booleanValue()) + booleanFromValue() = true and + booleanFromValue() = false } predicate notClass() { diff --git a/python/ql/src/semmle/python/types/Version.qll b/python/ql/src/semmle/python/types/Version.qll index 401b22b27ede..5d8e715f5379 100644 --- a/python/ql/src/semmle/python/types/Version.qll +++ b/python/ql/src/semmle/python/types/Version.qll @@ -1,6 +1,4 @@ import python -import semmle.python.GuardedControlFlow -private import semmle.python.pointsto.PointsTo /** A Version of the Python interpreter. * Currently only 2.7 or 3.x but may include different sets of versions in the future. */ @@ -18,152 +16,3 @@ class Version extends int { } } - -Object theSysVersionInfoTuple() { - py_cmembers_versioned(theSysModuleObject(), "version_info", result, major_version().toString()) -} - -Object theSysHexVersionNumber() { - py_cmembers_versioned(theSysModuleObject(), "hexversion", result, major_version().toString()) -} - -Object theSysVersionString() { - py_cmembers_versioned(theSysModuleObject(), "version", result, major_version().toString()) -} - - -string reversed(Cmpop op) { - op instanceof Lt and result = ">" - or - op instanceof Gt and result = "<" - or - op instanceof GtE and result = "<=" - or - op instanceof LtE and result = ">=" - or - op instanceof Eq and result = "==" - or - op instanceof NotEq and result = "!=" -} - - -/** DEPRECATED: - * A test on the major version of the Python interpreter - * */ -class VersionTest extends @py_flow_node { - - string toString() { - result = "VersionTest" - } - - VersionTest() { - PointsTo::version_const(this, _, _) - } - - predicate isTrue() { - PointsTo::version_const(this, _, true) - } - - AstNode getNode() { - result = this.(ControlFlowNode).getNode() - } - -} - -/** A guard on the major version of the Python interpreter */ -class VersionGuard extends ConditionBlock { - - VersionGuard() { - exists(VersionTest v | - PointsTo::points_to(this.getLastNode(), _, v, _, _) or - PointsTo::points_to(this.getLastNode(), _, _, _, v) - ) - } - - predicate isTrue() { - exists(VersionTest v | - v.isTrue() | - PointsTo::points_to(this.getLastNode(), _, v, _, _) or - PointsTo::points_to(this.getLastNode(), _, _, _, v) - ) - } - -} - -string os_name(StrConst s) { - exists(string t | - t = s.getText() | - (t = "linux" or t = "linux2") and result = "Linux" - or - t = "win32" and result = "Windows" - or - t = "darwin" and result = "Darwin" - or - not t = "linux" and not t = "linux2" and not t = "win32" and not t = "darwin" and result = t - ) -} - -predicate get_platform_name(Expr e) { - exists(Attribute a, Name n | a = e and n = a.getObject() | - n.getId() = "sys" and a.getName() = "platform" - ) - or - exists(Call c, Attribute a, Name n | - c = e and a = c.getFunc() and n = a.getObject() | - a.getName() = "system" and n.getId() = "platform" - ) -} - -predicate os_compare(ControlFlowNode f, string name) { - exists(Compare c, Expr l, Expr r, Cmpop op | - c = f.getNode() and - l = c.getLeft() and - r = c.getComparator(0) and - op = c.getOp(0) | - (op instanceof Eq or op instanceof Is) - and - ( get_platform_name(l) and name = os_name(r) - or - get_platform_name(r) and name = os_name(l) - ) - ) -} - -class OsTest extends @py_flow_node { - - OsTest() { - os_compare(this, _) - } - - string getOs() { - os_compare(this, result) - } - - string toString() { - result = "OsTest" - } - - AstNode getNode() { - result = this.(ControlFlowNode).getNode() - } - -} - - -class OsGuard extends ConditionBlock { - - OsGuard() { - exists(OsTest t | - PointsTo::points_to(this.getLastNode(), _, _, theBoolType(), t) - ) - } - - string getOs() { - exists(OsTest t | - PointsTo::points_to(this.getLastNode(), _, _, theBoolType(), t) and result = t.getOs() - ) - } - -} - - From 12853ccf30dc395b9333f5c06a25f467b018da06 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 25 Mar 2019 13:14:19 +0000 Subject: [PATCH 012/128] Python points-to: Add support for tuples. --- .../semmle/python/objects/ObjectInternal.qll | 1 + .../src/semmle/python/objects/Sequences.qll | 135 ++++++++++++++++++ .../ql/src/semmle/python/objects/TObject.qll | 10 ++ 3 files changed, 146 insertions(+) create mode 100644 python/ql/src/semmle/python/objects/Sequences.qll diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 8e26d3f330c1..94206fa69f3c 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -9,6 +9,7 @@ import semmle.python.objects.Classes import semmle.python.objects.Instances import semmle.python.objects.Callables import semmle.python.objects.Constants +import semmle.python.objects.Sequences class ObjectInternal extends TObject { diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll new file mode 100644 index 000000000000..2ae6a57203ac --- /dev/null +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -0,0 +1,135 @@ + +import python + + + + +private import semmle.python.objects.TObject +private import semmle.python.objects.ObjectInternal +private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsToContext +private import semmle.python.pointsto.MRO2 +private import semmle.python.types.Builtins + +abstract class SequenceObjectInternal extends ObjectInternal { + + abstract ObjectInternal getItem(int n); + + abstract int length(); + + /** The boolean value of this object, this may be both + * true and false if the "object" represents a set of possible objects. */ + override boolean booleanValue() { + this.length() = 0 and result = false + or + this.length() != 0 and result = true + } + +} + +abstract class TupleObjectInternal extends SequenceObjectInternal { + + override string toString() { + result = "tuple()" + } + + /** Gets the class declaration for this object, if it is a declared class. */ + override ClassDecl getClassDeclaration() { none() } + + /** True if this "object" is a class. */ + override boolean isClass() { result = false } + + override ObjectInternal getClass() { result = ObjectInternal::builtin("tuple") } + + /** True if this "object" can be meaningfully analysed for + * truth or false in comparisons. For example, `None` or `int` can be, but `int()` + * or an unknown string cannot. + */ + override boolean isComparable() { result = true } + + /** Holds if `obj` is the result of calling `this` and `origin` is + * the origin of `obj`. + */ + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } + + /** Holds if `obj` is the result of calling `this` and `origin` is + * the origin of `obj` with callee context `callee`. + */ + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } + + /** The integer value of things that have integer values. + * That is, ints and bools. + */ + override int intValue() { none() } + + /** The integer value of things that have integer values. + * That is, strings. + */ + override string strValue() { none() } + + override predicate calleeAndOffset(Function scope, int paramOffset) { none() } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate attributesUnknown() { none() } + +} + +class BuiltinTupleObjectInternal extends TBuiltinTuple, TupleObjectInternal { + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + none() + } + + override Builtin getBuiltin() { + this = TBuiltinTuple(result) + } + + override ControlFlowNode getOrigin() { + none() + } + + override ObjectInternal getItem(int n) { + result.getBuiltin() = this.getBuiltin().getItem(n) + } + + override int length() { + exists(Builtin b | + b = this.getBuiltin() and + result = count(int n | exists(b.getItem(n))) + ) + } +} + + +class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal { + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + this = TPythonTuple(node, context) + } + + override Builtin getBuiltin() { + none() + } + + override ControlFlowNode getOrigin() { + this = TPythonTuple(result, _) + } + + override ObjectInternal getItem(int n) { + exists(TupleNode t, PointsToContext context | + this = TPythonTuple(t, context) and + PointsTo2::points_to(t.getElement(n), context, result, _) + ) + } + + override int length() { + exists(TupleNode t | + this = TPythonTuple(t, _) and + result = count(int n | exists(t.getElement(n))) + ) + } + +} + + diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index cba68bd2e728..4cfbff2c9c56 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -18,6 +18,7 @@ newtype TObject = TBuiltinOpaqueObject(Builtin bltn) { not bltn.isClass() and not bltn.isFunction() and not bltn.isMethod() and not bltn.isModule() and + not bltn.getClass() = Builtin::special("tuple") and not exists(bltn.intValue()) and not exists(bltn.strValue()) and not py_special_objects(bltn, _) @@ -99,6 +100,14 @@ newtype TObject = TStaticMethod(CallNode instantiation, CallableObjectInternal function) { static_method(instantiation, function, _) } + or + TBuiltinTuple(Builtin bltn) { + bltn.getClass() = Builtin::special("tuple") + } + or + TPythonTuple(TupleNode origin, PointsToContext context) { + context.appliesTo(origin) + } private predicate is_power_2(int n) { n = 1 or @@ -254,6 +263,7 @@ library class ClassDecl extends @py_object { exists(string name | this = Builtin::special(name) | not name = "object" and + not name = "list" and not name = "set" and not name.matches("%Exception") and not name.matches("%Error") From 85a9016c8ce55c5f7190b50ab0fd9e778c26c248 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 25 Mar 2019 14:24:23 +0000 Subject: [PATCH 013/128] Python points-to: make 'self' instances distinct from other instances. --- .../src/semmle/python/objects/Instances.qll | 83 +++++++++++++++++++ .../ql/src/semmle/python/objects/TObject.qll | 11 +-- .../src/semmle/python/pointsto/PointsTo2.qll | 9 +- 3 files changed, 96 insertions(+), 7 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 71916c18a051..deb7c6ea95c1 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -83,6 +83,89 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { } +class SelfInstanceInternal extends TSelfInstance, ObjectInternal { + + override string toString() { + result = "self" + } + + /** The boolean value of this object, if it has one */ + override boolean booleanValue() { + //result = this.getClass().instancesBooleanValue() + result = maybe() + } + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + none() + } + + predicate parameterAndContext(ParameterDefinition def, PointsToContext context) { + this = TSelfInstance(def, context, _) + } + + /** Gets the class declaration for this object, if it is a declared class. */ + override ClassDecl getClassDeclaration() { + none() + } + + override boolean isClass() { result = false } + + override boolean isComparable() { result = false } + + override ObjectInternal getClass() { + this = TSelfInstance(_, _, result) + } + + /** Gets the `Builtin` for this object, if any. + * All objects (except unknown and undefined values) should return + * exactly one result for either this method or `getOrigin()`. + */ + override Builtin getBuiltin() { + none() + } + + /** Gets a control flow node that represents the source origin of this + * objects. + * All objects (except unknown and undefined values) should return + * exactly one result for either this method or `getBuiltin()`. + */ + override ControlFlowNode getOrigin() { + exists(ParameterDefinition def | + this = TSelfInstance(def, _, _) and + result = def.getDefiningNode() + ) + } + + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + // In general instances aren't callable, but some are... + // TO DO -- Handle cases where class overrides __call__ + none() + } + + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + override predicate attributesUnknown() { any() } + +} + /** Represents a value that has a known class, but no other information */ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 4cfbff2c9c56..df10ee187c6f 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -78,9 +78,10 @@ newtype TObject = TSpecificInstance(ControlFlowNode instantiation, ClassObjectInternal cls, PointsToContext context) { PointsTo2::points_to(instantiation.(CallNode).getFunction(), context, cls, _) and cls.isSpecial() = false - or - // Self - self_parameter(instantiation.getNode(), context, cls) + } + or + TSelfInstance(ParameterDefinition def, PointsToContext context, PythonClassObjectInternal cls) { + self_parameter(def, context, cls) } or TBoundMethod(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext context) { @@ -169,10 +170,10 @@ predicate receiver(AttrNode instantiation, PointsToContext context, ObjectIntern /** Helper self parameters: `def meth(self, ...): ...`. */ pragma [noinline] -private predicate self_parameter(Parameter def, PointsToContext context, PythonClassObjectInternal cls) { +private predicate self_parameter(ParameterDefinition def, PointsToContext context, PythonClassObjectInternal cls) { def.isSelf() and exists(Function scope | - def.(Name).getScope() = scope and + def.getScope() = scope and def.isSelf() and context.isRuntime() and context.appliesToScope(scope) and scope.getScope() = cls.getScope() and diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index 39ccc2619fa1..9e9abf2fbe8f 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -305,8 +305,8 @@ cached module PointsTo2 { or assignment_points_to(def, context, value, origin) //// TO DO... - // or - // self_parameter_points_to(def, context, value, origin) + or + self_parameter_points_to(def, context, value, origin) or delete_points_to(def, context, value, origin) or @@ -344,6 +344,11 @@ cached module PointsTo2 { ssa_variable_points_to(def.getInput(), context, value, origin) } + private predicate self_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + origin = CfgOrigin::fromCfgNode(def.getDefiningNode()) and + value.(SelfInstanceInternal).parameterAndContext(def, context) + } + /** Holds if ESSA edge refinement, `def`, refers to `(value, cls, origin)`. */ private predicate ssa_filter_definition_points_to(PyEdgeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(ControlFlowNode test, ControlFlowNode use | From 48297e299e60149f94f650d9da803309c8b79c3f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 25 Mar 2019 15:17:03 +0000 Subject: [PATCH 014/128] Python points-to: Improve handling of 'type' object. --- .../src/semmle/python/objects/Callables.qll | 23 +++----- .../ql/src/semmle/python/objects/Classes.qll | 53 ++++++++++++++++++- .../ql/src/semmle/python/objects/Modules.qll | 4 +- .../semmle/python/objects/ObjectInternal.qll | 2 + .../ql/src/semmle/python/objects/TObject.qll | 7 ++- 5 files changed, 70 insertions(+), 19 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index fcbedfba8185..6f8cecbcb825 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -80,10 +80,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti PointsTo2::points_to(rval, callee, obj, origin) ) } - - override predicate callResult(ObjectInternal obj, CfgOrigin origin) { - none() - } + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } override predicate calleeAndOffset(Function scope, int paramOffset) { scope = this.getScope() and paramOffset = 0 @@ -115,9 +112,7 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc override boolean isComparable() { result = true } - override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { - none() - } + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } override predicate callResult(ObjectInternal obj, CfgOrigin origin) { exists(Builtin func, ClassObjectInternal cls | @@ -191,9 +186,7 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod override boolean isComparable() { result = true } - override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { - none() - } + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } override predicate callResult(ObjectInternal obj, CfgOrigin origin) { // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. @@ -247,7 +240,7 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { } override predicate callResult(ObjectInternal obj, CfgOrigin origin) { - this.getFunction().callResult(obj, origin) + none() } override ControlFlowNode getOrigin() { @@ -295,10 +288,10 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { override ControlFlowNode getOrigin() { this = TClassMethod(result, _) } - override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } - override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } + override int intValue() { none() } override string strValue() { none() } @@ -344,10 +337,10 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { override ControlFlowNode getOrigin() { this = TStaticMethod(result, _) } - override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } - override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } + override int intValue() { none() } override string strValue() { none() } diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index cf1af47ffb2b..3e9bca1c9df2 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -140,7 +140,7 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec } override predicate callResult(ObjectInternal obj, CfgOrigin origin) { - // TO DO .. Result should (in most cases) be an instance + // Handled by Instance classes. none() } @@ -197,4 +197,55 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { } +class TypeInternal extends ClassObjectInternal, TType { + + override string toString() { + result = "type" + } + + override ClassDecl getClassDeclaration() { + result = Builtin::special("type") + } + + override ObjectInternal getClass() { + result = this + } + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + none() + } + + override boolean isComparable() { result = false } + override Builtin getBuiltin() { + result = Builtin::special("type") + } + + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { + exists(CallNode call, PointsToContext caller, ObjectInternal instance | + callee.fromCall(call, caller) | + count(call.getAnArg()) = 1 and + PointsTo2::points_to(call.getArg(0), caller, instance, origin) and + obj = instance.getClass() + ) + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + none() + } + + override ControlFlowNode getOrigin() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + override predicate attributesUnknown() { any() } + +} diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 862a98e64321..a42fe2d604c6 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -12,12 +12,12 @@ abstract class ModuleObjectInternal extends ObjectInternal { abstract Module getSourceModule(); - override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { // Modules aren't callable none() } - override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { // Modules aren't callable none() } diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 94206fa69f3c..b8c569d22be6 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -311,6 +311,8 @@ module ObjectInternal { result = TBuiltinFunctionObject(Builtin::builtin(name)) or result = TBuiltinOpaqueObject(Builtin::builtin(name)) + or + name = "type" and result = TType() } ObjectInternal sysModules() { diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index df10ee187c6f..5a5093cd182b 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -6,7 +6,9 @@ private import semmle.python.pointsto.PointsToContext newtype TObject = TBuiltinClassObject(Builtin bltn) { - bltn.isClass() and not bltn = Builtin::unknownType() + bltn.isClass() and + not bltn = Builtin::unknownType() and + not bltn = Builtin::special("type") } or TBuiltinFunctionObject(Builtin bltn) { bltn.isFunction() } @@ -109,6 +111,8 @@ newtype TObject = TPythonTuple(TupleNode origin, PointsToContext context) { context.appliesTo(origin) } + or + TType() private predicate is_power_2(int n) { n = 1 or @@ -266,6 +270,7 @@ library class ClassDecl extends @py_object { not name = "object" and not name = "list" and not name = "set" and + not name = "dict" and not name.matches("%Exception") and not name.matches("%Error") ) From f5c32421f454590cec61b73cd564a6d37ee429ea Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 25 Mar 2019 16:44:02 +0000 Subject: [PATCH 015/128] Python points-to: Handle list, dict and float literals as instances. --- .../ql/src/semmle/python/objects/TObject.qll | 35 +++++++++++++++---- .../src/semmle/python/pointsto/PointsTo2.qll | 2 +- python/ql/src/semmle/python/types/Object.qll | 3 +- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 5a5093cd182b..8832d3d340f2 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -80,6 +80,8 @@ newtype TObject = TSpecificInstance(ControlFlowNode instantiation, ClassObjectInternal cls, PointsToContext context) { PointsTo2::points_to(instantiation.(CallNode).getFunction(), context, cls, _) and cls.isSpecial() = false + or + literal_instantiation(instantiation, cls, context) } or TSelfInstance(ParameterDefinition def, PointsToContext context, PythonClassObjectInternal cls) { @@ -129,6 +131,19 @@ predicate class_method(CallNode instantiation, CallableObjectInternal function, PointsTo2::points_to(instantiation.getArg(0), context, function, _) } +predicate literal_instantiation(ControlFlowNode n, ClassObjectInternal cls, PointsToContext context) { + context.appliesTo(n) and + ( + n instanceof ListNode and cls = ObjectInternal::builtin("list") + or + n instanceof DictNode and cls = ObjectInternal::builtin("dict") + or + n.getNode() instanceof FloatLiteral and cls = ObjectInternal::builtin("float") + or + n.getNode() instanceof ImaginaryLiteral and cls = ObjectInternal::builtin("complex") + ) +} + predicate super_instantiation(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext context) { PointsTo2::points_to(instantiation.getFunction(), context, ObjectInternal::builtin("super"), _) and ( @@ -267,12 +282,20 @@ library class ClassDecl extends @py_object { predicate isSpecial() { exists(string name | this = Builtin::special(name) | - not name = "object" and - not name = "list" and - not name = "set" and - not name = "dict" and - not name.matches("%Exception") and - not name.matches("%Error") + name = "type" or + name = "bool" or + name = "NoneType" or + name = "int" or + name = "long" or + name = "str" or + name = "bytes" or + name = "unicode" or + name = "tuple" or + name = "property" or + name = "classmethod" or + name = "staticmethod" or + name = "MethodType" or + name = "ModuleType" ) } diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index 9e9abf2fbe8f..34cefd5b339f 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -166,7 +166,7 @@ cached module PointsTo2 { /* Holds if the edge `pred` -> `succ` is reachable, given the context `context`. */ pragma [noopt] - private predicate controlledReachableEdge(BasicBlock pred, BasicBlock succ, PointsToContext context) { + cached predicate controlledReachableEdge(BasicBlock pred, BasicBlock succ, PointsToContext context) { exists(ConditionBlock guard, ObjectInternal value, boolean sense, ControlFlowNode test | test = guard.getLastNode() and points_to(test, context, value, _) and diff --git a/python/ql/src/semmle/python/types/Object.qll b/python/ql/src/semmle/python/types/Object.qll index 37315eefa627..8d57865c492e 100644 --- a/python/ql/src/semmle/python/types/Object.qll +++ b/python/ql/src/semmle/python/types/Object.qll @@ -6,7 +6,8 @@ private import semmle.python.types.Builtins private cached predicate is_an_object(@py_object obj) { /* CFG nodes for numeric literals, all of which have a @py_cobject for the value of that literal */ obj instanceof ControlFlowNode and - not obj.(ControlFlowNode).getNode() instanceof ImmutableLiteral + not obj.(ControlFlowNode).getNode() instanceof IntegerLiteral and + not obj.(ControlFlowNode).getNode() instanceof StrConst or obj instanceof Builtin } From dbf228d005381b0f928b9898bed5e70b0b5e121c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 25 Mar 2019 17:33:01 +0000 Subject: [PATCH 016/128] Python points-to: Better handling of *args, **kwargs and procedures. --- .../src/semmle/python/objects/Callables.qll | 17 +++++++++-- .../src/semmle/python/objects/Constants.qll | 2 +- .../semmle/python/objects/ObjectInternal.qll | 1 - .../src/semmle/python/pointsto/PointsTo2.qll | 30 +++++++++++++++++-- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 6f8cecbcb825..af6113798bd1 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -75,12 +75,25 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { exists(Function func, ControlFlowNode rval | func = this.getScope() and - callee.appliesToScope(func) and + callee.appliesToScope(func) | rval = func.getAReturnValueFlowNode() and PointsTo2::points_to(rval, callee, obj, origin) + or + exists(Return ret | + ret.getScope() = func and + PointsTo2::reachableBlock(ret.getAFlowNode().getBasicBlock(), callee) and + not exists(ret.getValue()) and + obj = ObjectInternal::none_() and + origin = CfgOrigin::unknown() + ) ) } - override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + this.getScope().isProcedure() and + obj = ObjectInternal::none_() and + origin = CfgOrigin::unknown() + } override predicate calleeAndOffset(Function scope, int paramOffset) { scope = this.getScope() and paramOffset = 0 diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index d5aa4a790c01..f132da6deabd 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -140,7 +140,7 @@ class NoneObjectInternal extends ObjectInternal, TNone { } override Builtin getBuiltin() { - none() + result = Builtin::special("None") } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index b8c569d22be6..1475f75b04c0 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -292,7 +292,6 @@ module ObjectInternal { result = TNone() } - ObjectInternal unknown() { result = TUnknown() } diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index 34cefd5b339f..30dde626838b 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -520,7 +520,7 @@ module InterModulePointsTo { exists(string name, ModuleObjectInternal mod, CfgOrigin orig | from_import_imports(f, context, mod, name) and (mod.getSourceModule() != f.getEnclosingModule() or mod.isBuiltin()) and - mod.attribute(name, value, origin) and + mod.attribute(name, value, orig) and origin = orig.asCfgNodeOrHere(f) // TO DO... $ variables. //mod.getSourceModule() = f.getEnclosingModule() and @@ -697,8 +697,8 @@ module InterProceduralPointsTo { named_parameter_points_to(def, context, value, origin) or default_parameter_points_to(def, context, value, origin) - // or - // special_parameter_points_to(def, context, value, origin) + or + special_parameter_points_to(def, context, value, origin) } /** Helper for `parameter_points_to` */ @@ -745,6 +745,30 @@ module InterProceduralPointsTo { ) } + /** Helper for parameter_points_to */ + pragma [noinline] + private predicate special_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + context.isRuntime() and + origin = def.getDefiningNode() and + exists(ControlFlowNode param | + param = def.getDefiningNode() | + exists(Function func | func.getVararg() = param.getNode()) and value = TUnknownInstance(ObjectInternal::builtin("tuple")) + or + exists(Function func | func.getKwarg() = param.getNode()) and value = TUnknownInstance(ObjectInternal::builtin("dict")) + ) + or + exists(PointsToContext caller, CallNode call, Function f, Parameter p | + context.fromCall(call, caller) and + context.appliesToScope(f) and + f.getAnArg() = p and p = def.getParameter() and + not p.isSelf() and + not exists(call.getArg(p.getPosition())) and + not exists(call.getArgByName(p.getName())) and + (exists(call.getNode().getKwargs()) or exists(call.getNode().getStarargs())) and + value = ObjectInternal::unknown() and origin = def.getDefiningNode() + ) + } + /** Holds if the `(argument, caller)` pair matches up with `(param, callee)` pair across call. */ cached predicate callsite_argument_transfer(ControlFlowNode argument, PointsToContext caller, ParameterDefinition param, PointsToContext callee) { exists(CallNode call, Function func, int n, int offset | From 0b0a6337f3a071d500cfa989b120f84c1fa7065c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 25 Mar 2019 18:23:48 +0000 Subject: [PATCH 017/128] Python points-to: Support descriptor protocols, particularly functions. --- .../src/semmle/python/objects/Callables.qll | 78 ++++++++++++++-- .../ql/src/semmle/python/objects/Classes.qll | 11 +++ .../src/semmle/python/objects/Constants.qll | 25 ++++++ .../src/semmle/python/objects/Instances.qll | 88 +++++++++++++++++-- .../ql/src/semmle/python/objects/Modules.qll | 7 ++ .../semmle/python/objects/ObjectInternal.qll | 29 ++++++ .../src/semmle/python/objects/Sequences.qll | 6 ++ .../ql/src/semmle/python/objects/TObject.qll | 16 ++-- .../src/semmle/python/pointsto/PointsTo2.qll | 19 +++- .../python/pointsto/PointsToContext.qll | 8 +- .../semmle/python/types/FunctionObject.qll | 2 +- 11 files changed, 260 insertions(+), 29 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index af6113798bd1..cf64ebee001a 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -38,12 +38,16 @@ abstract class CallableObjectInternal extends ObjectInternal { override predicate attributesUnknown() { none() } + abstract Function getScope(); + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + } class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFunctionObject { - Function getScope() { + override Function getScope() { exists(CallableExpr expr | this = TPythonFunctionObject(expr.getAFlowNode()) and result = expr.getInnerScope() @@ -51,7 +55,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti } override string toString() { - result = this.getScope().toString() + result = "Function " + this.getScope().getQualifiedName() } override predicate introduced(ControlFlowNode node, PointsToContext context) { @@ -103,6 +107,12 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti result = this.getScope().getName() } + override boolean isDescriptor() { result = true } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { + value = TBoundMethod(instance, this) and origin = CfgOrigin::unknown() + } + } class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunctionObject { @@ -176,6 +186,13 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc ) } + override Function getScope() { none() } + + override boolean isDescriptor() { result = false } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + } @@ -218,6 +235,19 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod result = this.getBuiltin().getName() } + override Function getScope() { none() } + + override boolean isDescriptor() { result = true } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { + instance.isClass() = false and + value = TBoundMethod(instance, this) and origin = CfgOrigin::unknown() + or + any(ObjectInternal obj).binds(instance, _, this) and + instance.isClass() = true and + value = this and origin = CfgOrigin::unknown() + } + } class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { @@ -227,15 +257,15 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { } CallableObjectInternal getFunction() { - this = TBoundMethod(_, _, result, _) + this = TBoundMethod(_, result) } ObjectInternal getSelf() { - this = TBoundMethod(_, result, _, _) + this = TBoundMethod(result, _) } override string toString() { - result = "bound method '" + this.getFunction().getName() + "' of " + this.getSelf().toString() + result = "Method(" + this.getFunction() + ", " + this.getSelf() + ")" } override ObjectInternal getClass() { @@ -243,7 +273,7 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { } override predicate introduced(ControlFlowNode node, PointsToContext context) { - this = TBoundMethod(node, _, _, context) + none() } override boolean isComparable() { result = false } @@ -253,7 +283,7 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { } override predicate callResult(ObjectInternal obj, CfgOrigin origin) { - none() + this.getFunction().callResult(obj, origin) } override ControlFlowNode getOrigin() { @@ -268,6 +298,15 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { result = this.getFunction().getName() } + + override Function getScope() { + result = this.getFunction().getScope() + } + + override boolean isDescriptor() { result = false } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + } class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { @@ -317,6 +356,22 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { override predicate attributesUnknown() { none() } + override boolean isDescriptor() { result = true } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { + any(ObjectInternal obj).binds(instance, _, this) and + ( + instance.isClass() = false and + value = TBoundMethod(instance.getClass(), this.getFunction()) + or + instance.isClass() = true and + value = TBoundMethod(instance, this.getFunction()) + ) and + origin = CfgOrigin::unknown() + } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + } class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { @@ -366,6 +421,15 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { override predicate attributesUnknown() { none() } + override boolean isDescriptor() { result = true } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { + any(ObjectInternal obj).binds(instance, _, this) and + value = this.getFunction() and origin = CfgOrigin::unknown() + } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + } diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 3e9bca1c9df2..1c5a88f51d60 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -33,6 +33,17 @@ abstract class ClassObjectInternal extends ObjectInternal { result = Types::getMro(this).isSpecial() } + override boolean isDescriptor() { result = false } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { + instance = this and + PointsTo2::attributeRequired(this, name) and + this.attribute(name, descriptor, _) and + descriptor.isDescriptor() = true + } + } class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject { diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index f132da6deabd..4363c312ecfa 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -3,6 +3,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.MRO2 private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins @@ -54,6 +55,12 @@ abstract class BooleanObjectInternal extends ObjectInternal { override predicate attributesUnknown() { none() } + override boolean isDescriptor() { result = false } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + } class TrueObjectInternal extends BooleanObjectInternal, TTrue { @@ -179,6 +186,12 @@ class NoneObjectInternal extends ObjectInternal, TNone { result = Builtin::special("None") } + override boolean isDescriptor() { result = false } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + } @@ -252,6 +265,12 @@ class IntObjectInternal extends ObjectInternal, TInt { result.(Builtin).intValue() = this.intValue() } + override boolean isDescriptor() { result = false } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + } @@ -324,6 +343,12 @@ class StringObjectInternal extends ObjectInternal, TString { result.(Builtin).strValue() = this.strValue() } + override boolean isDescriptor() { result = false } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + } diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index deb7c6ea95c1..9a58e55210de 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -4,6 +4,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.MRO2 private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins @@ -76,17 +77,43 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { } override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - none() + PointsTo2::attributeRequired(this, name) and + instance_getattr(this, Types::getMro(this.getClass()), name, value, origin) } override predicate attributesUnknown() { any() } + override boolean isDescriptor() { result = false } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { + this = instance and descriptor.isDescriptor() = true and + exists(AttrNode attr | + PointsTo2::points_to(attr.getObject(name), _, instance, _) and + this.getClass().attribute(name, descriptor, _) + ) + } + +} + +bindingset[instance, mro, name] +predicate instance_getattr(ObjectInternal instance, ClassList mro, string name, ObjectInternal value, CfgOrigin origin) { + exists(ObjectInternal descriptor, CfgOrigin desc_origin | + Types::declaredAttribute(mro.findDeclaringClass(name), name, descriptor, desc_origin) | + descriptor.isDescriptor() = false and + value = descriptor and origin = desc_origin + or + descriptor.isDescriptor() = true and + descriptor.descriptorGet(instance, value, origin) + ) } + class SelfInstanceInternal extends TSelfInstance, ObjectInternal { override string toString() { - result = "self" + result = "self instance of " + this.getClass().(ClassObjectInternal).getName() } /** The boolean value of this object, if it has one */ @@ -159,11 +186,25 @@ class SelfInstanceInternal extends TSelfInstance, ObjectInternal { } override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - none() + PointsTo2::attributeRequired(this, name) and + instance_getattr(this, Types::getMro(this.getClass()), name, value, origin) } override predicate attributesUnknown() { any() } + override boolean isDescriptor() { result = false } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { + descriptor.isDescriptor() = true and + this = instance and + exists(AttrNode attr | + PointsTo2::points_to(attr.getObject(name), _, this, _) and + this.getClass().attribute(name, descriptor, _) + ) + } + } /** Represents a value that has a known class, but no other information */ @@ -238,18 +279,32 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { } override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - none() + PointsTo2::attributeRequired(this, name) and + instance_getattr(this, Types::getMro(this.getClass()), name, value, origin) } override predicate attributesUnknown() { any() } + override boolean isDescriptor() { result = false } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { + descriptor.isDescriptor() = true and + this = instance and + exists(AttrNode attr | + PointsTo2::points_to(attr.getObject(name), _, this, _) and + this.getClass().attribute(name, descriptor, _) + ) + } + } class SuperInstance extends TSuperInstance, ObjectInternal { override string toString() { - result = "super()" + result = "super(" + this.getStartClass().toString() + ", " + this.getSelf().toString() + ")" } override boolean booleanValue() { result = true } @@ -295,12 +350,29 @@ class SuperInstance extends TSuperInstance, ObjectInternal { override predicate calleeAndOffset(Function scope, int paramOffset) { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } - override predicate attributesUnknown() { none() } -} + override boolean isDescriptor() { result = false } + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + PointsTo2::attributeRequired(this, name) and + instance_getattr(this.getSelf(), this.getMro(), name, value, origin) + } + + private ClassList getMro() { + result = Types::getMro(this.getSelf().getClass()).startingAt(this.getStartClass()).getTail() + } + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { + descriptor.isDescriptor() = true and + exists(AttrNode attr | + PointsTo2::points_to(attr.getObject(name), _, this, _) and + instance = this.getSelf() and + Types::declaredAttribute(this.getMro().findDeclaringClass(name), name, descriptor, _) + ) + } + +} diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index a42fe2d604c6..b80644811fd8 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -3,6 +3,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.MRO2 private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins @@ -38,6 +39,12 @@ abstract class ModuleObjectInternal extends ObjectInternal { result = ObjectInternal::moduleType() } + override boolean isDescriptor() { result = false } + + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } } class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleObject { diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 1475f75b04c0..2966fae9169d 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -2,6 +2,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.MRO2 private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins import semmle.python.objects.Modules @@ -86,6 +87,16 @@ class ObjectInternal extends TObject { result = this.getBuiltin() } + abstract boolean isDescriptor(); + + abstract predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin); + + /** Holds if attribute lookup on this object may "bind" `instance` to `descriptor`. + * Here "bind" means that `instance` is passed to the `descriptor.__get__()` method + * at runtime. The term "bind" is used as this most likely results in a bound-method. + */ + abstract predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor); + } @@ -151,6 +162,12 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { override predicate attributesUnknown() { none() } + override boolean isDescriptor() { result = false } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + } @@ -214,6 +231,12 @@ class UnknownInternal extends ObjectInternal, TUnknown { override predicate attributesUnknown() { any() } + override boolean isDescriptor() { result = false } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + } class UndefinedInternal extends ObjectInternal, TUndefined { @@ -278,6 +301,12 @@ class UndefinedInternal extends ObjectInternal, TUndefined { override predicate attributesUnknown() { none() } + override boolean isDescriptor() { none() } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + } module ObjectInternal { diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index 2ae6a57203ac..3f022ad051a4 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -25,6 +25,12 @@ abstract class SequenceObjectInternal extends ObjectInternal { this.length() != 0 and result = true } + override boolean isDescriptor() { result = false } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + } abstract class TupleObjectInternal extends SequenceObjectInternal { diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 8832d3d340f2..d69a56636f1e 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -88,11 +88,12 @@ newtype TObject = self_parameter(def, context, cls) } or - TBoundMethod(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext context) { - method_binding(instantiation, self, function, context) + TBoundMethod(ObjectInternal self, CallableObjectInternal function) { + any(ObjectInternal obj).binds(self, _, function) and + function.isDescriptor() = true } or - TUnknownInstance(BuiltinClassObjectInternal cls) { any() } + TUnknownInstance(BuiltinClassObjectInternal cls) { cls != ObjectInternal::builtin("super") } or TSuperInstance(ObjectInternal self, ClassObjectInternal startclass) { super_instantiation(_, self, startclass, _) @@ -162,7 +163,10 @@ predicate super_instantiation(CallNode instantiation, ObjectInternal self, Class ) } + +bindingset[self, function] predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext context) { + exists(ObjectInternal obj, string name | receiver(instantiation, context, obj, name) | exists(ObjectInternal cls | @@ -172,9 +176,10 @@ predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableOb self = obj ) or - exists(SuperInstance sup | + exists(SuperInstance sup, ClassObjectInternal decl | sup = obj and - sup.getStartClass().attribute(name, function, _) and + decl = Types::getMro(self.getClass()).startingAt(sup.getStartClass()).findDeclaringClass(name) and + Types::declaredAttribute(decl, name, function, _) and self = sup.getSelf() ) ) @@ -283,6 +288,7 @@ library class ClassDecl extends @py_object { exists(string name | this = Builtin::special(name) | name = "type" or + name = "super" or name = "bool" or name = "NoneType" or name = "int" or diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index 30dde626838b..79e2a2eaa8e8 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -129,6 +129,18 @@ cached module PointsTo2 { // f.(CustomPointsToFact).pointsTo(context, value, origin) } + /** Holds if the attribute `name` is required for `obj` */ + cached predicate attributeRequired(ObjectInternal obj, string name) { + points_to(any(AttrNode a).getObject(name), _, obj, _) + or + exists(CallNode call, PointsToContext ctx, StringObjectInternal nameobj | + points_to(call.getFunction(), ctx, ObjectInternal::builtin("getattr"), _) and + points_to(call.getArg(0), ctx, obj, _) and + points_to(call.getArg(1), ctx, nameobj, _) and + nameobj.strValue() = name + ) + } + cached CallNode get_a_call(ObjectInternal func, PointsToContext context) { points_to(result.getFunction(), context, func, _) } @@ -261,14 +273,13 @@ cached module PointsTo2 { /** Holds if `f` is an attribute `x.attr` and points to `(value, cls, origin)`. */ pragma [noinline] private predicate attribute_load_points_to(AttrNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + f.isLoad() and exists(ObjectInternal object, string name, CfgOrigin orig | points_to(f.getObject(name), context, object, _) | object.attribute(name, value, orig) and origin = orig.fix(f) - ) - or - exists(ObjectInternal object | - points_to(f.getObject(), context, object, _) and + or + object.attributesUnknown() and origin = f and value = ObjectInternal::unknown() ) // TO DO -- Support CustomPointsToAttribute diff --git a/python/ql/src/semmle/python/pointsto/PointsToContext.qll b/python/ql/src/semmle/python/pointsto/PointsToContext.qll index 3558ef4a177a..592e3c06b348 100644 --- a/python/ql/src/semmle/python/pointsto/PointsToContext.qll +++ b/python/ql/src/semmle/python/pointsto/PointsToContext.qll @@ -169,10 +169,10 @@ class PointsToContext extends TPointsToContext { this = TRuntimeContext() and executes_in_runtime_context(s) or /* Called functions, regardless of their name */ - exists(PythonFunctionObjectInternal func, ControlFlowNode call, TPointsToContext outerContext | - call = PointsTo2::get_a_call(func, outerContext) and - this = TCallContext(call, outerContext, _) and - s = func.getScope() + exists(CallableObjectInternal callable, ControlFlowNode call, TPointsToContext outerContext | + call = PointsTo2::get_a_call(callable, outerContext) and + this = TCallContext(call, outerContext, _) | + s = callable.getScope() ) or InterProceduralPointsTo::callsite_calls_function(_, _, s, this, _) diff --git a/python/ql/src/semmle/python/types/FunctionObject.qll b/python/ql/src/semmle/python/types/FunctionObject.qll index 4f1115a3997a..09889cfd4230 100644 --- a/python/ql/src/semmle/python/types/FunctionObject.qll +++ b/python/ql/src/semmle/python/types/FunctionObject.qll @@ -59,7 +59,7 @@ abstract class FunctionObject extends Object { /** Gets a call-site from where this function is called */ ControlFlowNode getACall() { - result = PointsTo2::get_a_call(theCallable(), _) + result = theCallable().getACall() } /** Gets a call-site from where this function is called, given the `context` */ From e9f58ba3a747419f0bac009bd2e5da9da7fc7c82 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 27 Mar 2019 11:20:44 +0000 Subject: [PATCH 018/128] Python: refactor ConstantObjects. --- .../src/semmle/python/objects/Constants.qll | 173 +++--------------- 1 file changed, 23 insertions(+), 150 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index 4363c312ecfa..222087bfe1a4 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -8,11 +8,8 @@ private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins -abstract class BooleanObjectInternal extends ObjectInternal { - BooleanObjectInternal() { - this = TTrue() or this = TFalse() - } +abstract class ConstantObjectInternal extends ObjectInternal { override ClassDecl getClassDeclaration() { none() @@ -22,22 +19,13 @@ abstract class BooleanObjectInternal extends ObjectInternal { override boolean isComparable() { result = true } - - override ObjectInternal getClass() { - result = TBuiltinClassObject(Builtin::special("bool")) - } - - override Builtin getBuiltin() { - none() - } - override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { - // Booleans aren't callable + // Constants aren't callable none() } override predicate callResult(ObjectInternal obj, CfgOrigin origin) { - // Booleans aren't callable + // Constants aren't callable none() } @@ -63,6 +51,19 @@ abstract class BooleanObjectInternal extends ObjectInternal { } +abstract class BooleanObjectInternal extends ConstantObjectInternal { + + BooleanObjectInternal() { + this = TTrue() or this = TFalse() + } + + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("bool")) + } + +} + class TrueObjectInternal extends BooleanObjectInternal, TTrue { override string toString() { @@ -85,7 +86,7 @@ class TrueObjectInternal extends BooleanObjectInternal, TTrue { none() } - override @py_object getSource() { + override Builtin getBuiltin() { result = Builtin::special("True") } @@ -113,13 +114,13 @@ class FalseObjectInternal extends BooleanObjectInternal, TFalse { none() } - override @py_object getSource() { + override Builtin getBuiltin() { result = Builtin::special("False") } } -class NoneObjectInternal extends ObjectInternal, TNone { +class NoneObjectInternal extends ConstantObjectInternal, TNone { override string toString() { result = "None" @@ -129,15 +130,6 @@ class NoneObjectInternal extends ObjectInternal, TNone { result = false } - override ClassDecl getClassDeclaration() { - none() - } - - override boolean isClass() { result = false } - - override boolean isComparable() { result = true } - - override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("NoneType")) } @@ -150,20 +142,6 @@ class NoneObjectInternal extends ObjectInternal, TNone { result = Builtin::special("None") } - override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { - // None isn't callable - none() - } - - override predicate callResult(ObjectInternal obj, CfgOrigin origin) { - // None isn't callable - none() - } - - override ControlFlowNode getOrigin() { - none() - } - override int intValue() { none() } @@ -172,30 +150,10 @@ class NoneObjectInternal extends ObjectInternal, TNone { none() } - override predicate calleeAndOffset(Function scope, int paramOffset) { - none() - } - - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - none() - } - - override predicate attributesUnknown() { none() } - - override @py_object getSource() { - result = Builtin::special("None") - } - - override boolean isDescriptor() { result = false } - - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } - - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } - } -class IntObjectInternal extends ObjectInternal, TInt { +class IntObjectInternal extends ConstantObjectInternal, TInt { override string toString() { result = "int " + this.intValue().toString() @@ -206,35 +164,12 @@ class IntObjectInternal extends ObjectInternal, TInt { node.getNode().(IntegerLiteral).getValue() = this.intValue() } - override ClassDecl getClassDeclaration() { - none() - } - - override boolean isClass() { result = false } - - override boolean isComparable() { result = true } - override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("int")) } - override Builtin getBuiltin() { - none() - } - - override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { - // ints aren't callable - none() - } - - override predicate callResult(ObjectInternal obj, CfgOrigin origin) { - // ints aren't callable - none() - } - - override ControlFlowNode getOrigin() { - none() + result.(Builtin).intValue() = this.intValue() } override int intValue() { @@ -251,30 +186,10 @@ class IntObjectInternal extends ObjectInternal, TInt { this.intValue() != 0 and result = true } - override predicate calleeAndOffset(Function scope, int paramOffset) { - none() - } - - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - none() - } - - override predicate attributesUnknown() { none() } - - override @py_object getSource() { - result.(Builtin).intValue() = this.intValue() - } - - override boolean isDescriptor() { result = false } - - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } - - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } - } -class StringObjectInternal extends ObjectInternal, TString { +class StringObjectInternal extends ConstantObjectInternal, TString { override string toString() { result = "'" + this.strValue() + "'" @@ -285,34 +200,12 @@ class StringObjectInternal extends ObjectInternal, TString { node.getNode().(StrConst).getText() = this.strValue() } - override ClassDecl getClassDeclaration() { - none() - } - - override boolean isClass() { result = false } - - override boolean isComparable() { result = true } - override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("unicode")) } override Builtin getBuiltin() { - none() - } - - override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { - // strings aren't callable - none() - } - - override predicate callResult(ObjectInternal obj, CfgOrigin origin) { - // strings aren't callable - none() - } - - override ControlFlowNode getOrigin() { - none() + result.(Builtin).strValue() = this.strValue() } override int intValue() { @@ -329,26 +222,6 @@ class StringObjectInternal extends ObjectInternal, TString { this.strValue() != "" and result = true } - override predicate calleeAndOffset(Function scope, int paramOffset) { - none() - } - - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - none() - } - - override predicate attributesUnknown() { none() } - - override @py_object getSource() { - result.(Builtin).strValue() = this.strValue() - } - - override boolean isDescriptor() { result = false } - - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } - - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } - } From 931100c77214c50d79bfab2db50de20a73af6df6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 27 Mar 2019 11:59:29 +0000 Subject: [PATCH 019/128] Python points-to: Add float objects for better backwards compatibility. --- .../src/semmle/python/objects/Constants.qll | 41 ++++++++++++++++++- .../src/semmle/python/objects/Instances.qll | 4 +- .../ql/src/semmle/python/objects/TObject.qll | 13 ++++-- .../ql/src/semmle/python/types/Builtins.qll | 5 +++ 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index 222087bfe1a4..f339ce4705dc 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -169,7 +169,7 @@ class IntObjectInternal extends ConstantObjectInternal, TInt { } override Builtin getBuiltin() { - result.(Builtin).intValue() = this.intValue() + result.intValue() = this.intValue() } override int intValue() { @@ -188,6 +188,45 @@ class IntObjectInternal extends ConstantObjectInternal, TInt { } +class FloatObjectInternal extends ConstantObjectInternal, TFloat { + + override string toString() { + result = "float " + this.floatValue().toString() + } + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + context.appliesTo(node) and + node.getNode().(FloatLiteral).getValue() = this.floatValue() + } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("float")) + } + + override Builtin getBuiltin() { + result.getName().toFloat() = this.floatValue() + } + + private float floatValue() { + this = TFloat(result) + } + + override int intValue() { + this = TFloat(result) + } + + override string strValue() { + none() + } + + override boolean booleanValue() { + this.floatValue() = 0.0 and result = false + or + this.floatValue() != 0.0 and result = true + } + +} + class StringObjectInternal extends ConstantObjectInternal, TString { diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 9a58e55210de..695f1d2b95ce 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -221,9 +221,7 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { } override predicate introduced(ControlFlowNode node, PointsToContext context) { - context.appliesTo(node) and - this.getClass() = ObjectInternal::builtin("float") and - node.getNode() instanceof FloatLiteral + none() } /** Gets the class declaration for this object, if it is a declared class. */ diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index d69a56636f1e..cce2ebce0cc4 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -22,6 +22,7 @@ newtype TObject = not bltn.isMethod() and not bltn.isModule() and not bltn.getClass() = Builtin::special("tuple") and not exists(bltn.intValue()) and + not exists(bltn.floatValue()) and not exists(bltn.strValue()) and not py_special_objects(bltn, _) } @@ -58,8 +59,8 @@ newtype TObject = // And all combinations of flags up to 2^8 n in [0..511] or // Any number explicitly mentioned in the source code. - exists(Num num | - n = num.getN().toInt() or + exists(IntegerLiteral num | + n = num.getValue() or exists(UnaryExpr neg | neg.getOp() instanceof USub and neg.getOperand() = num) and n = -num.getN().toInt() ) @@ -67,6 +68,10 @@ newtype TObject = n = any(Builtin b).intValue() } or + TFloat(float f) { + f = any(FloatLiteral num).getValue() + } + or TString(string s) { // Any string explicitly mentioned in the source code. s = any(StrConst str).getText() @@ -139,8 +144,6 @@ predicate literal_instantiation(ControlFlowNode n, ClassObjectInternal cls, Poin or n instanceof DictNode and cls = ObjectInternal::builtin("dict") or - n.getNode() instanceof FloatLiteral and cls = ObjectInternal::builtin("float") - or n.getNode() instanceof ImaginaryLiteral and cls = ObjectInternal::builtin("complex") ) } @@ -303,6 +306,8 @@ library class ClassDecl extends @py_object { name = "MethodType" or name = "ModuleType" ) + or + this = Builtin::builtin("float") } } diff --git a/python/ql/src/semmle/python/types/Builtins.qll b/python/ql/src/semmle/python/types/Builtins.qll index a61d7f9e4ad1..400ec15ab3e8 100644 --- a/python/ql/src/semmle/python/types/Builtins.qll +++ b/python/ql/src/semmle/python/types/Builtins.qll @@ -87,6 +87,11 @@ class Builtin extends @py_cobject { result = this.getName().toInt() } + float floatValue() { + (this.getClass() = Builtin::special("float")) and + result = this.getName().toFloat() + } + string strValue() { (this.getClass() = Builtin::special("unicode") or this.getClass() = Builtin::special("bytes")) and From d3762ac5a1fc6d273a38c04072c8e229f125aab9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 27 Mar 2019 12:57:39 +0000 Subject: [PATCH 020/128] Rename 'points_to' to 'pointsTo'. --- python/ql/src/semmle/python/Exprs.qll | 4 +- python/ql/src/semmle/python/Flow.qll | 6 +- .../src/semmle/python/objects/Callables.qll | 2 +- .../ql/src/semmle/python/objects/Classes.qll | 2 +- .../src/semmle/python/objects/Constants.qll | 2 +- .../src/semmle/python/objects/Instances.qll | 8 +- .../ql/src/semmle/python/objects/Modules.qll | 6 +- .../src/semmle/python/objects/ObjectAPI.qll | 8 +- .../src/semmle/python/objects/Sequences.qll | 2 +- .../ql/src/semmle/python/objects/TObject.qll | 22 +- .../src/semmle/python/pointsto/PointsTo2.qll | 286 +++++++++--------- 11 files changed, 174 insertions(+), 174 deletions(-) diff --git a/python/ql/src/semmle/python/Exprs.qll b/python/ql/src/semmle/python/Exprs.qll index 2af87d47d56a..5c36b0d8889d 100644 --- a/python/ql/src/semmle/python/Exprs.qll +++ b/python/ql/src/semmle/python/Exprs.qll @@ -86,7 +86,7 @@ class Expr extends Expr_, AstNode { */ predicate refersTo(Context context, Object obj, ClassObject cls, AstNode origin) { exists(Value value, ControlFlowNode cfgorigin | - PointsTo2::points_to(this.getAFlowNode(), context, value, cfgorigin) and + PointsTo2::pointsTo(this.getAFlowNode(), context, value, cfgorigin) and origin.getAFlowNode() = cfgorigin and cls = value.getClass().getSource() | if exists(value.getSource()) then @@ -102,7 +102,7 @@ class Expr extends Expr_, AstNode { */ predicate refersTo(Object obj, AstNode origin) { exists(Value value, ControlFlowNode cfgorigin | - PointsTo2::points_to(this.getAFlowNode(), _, value, cfgorigin) and + PointsTo2::pointsTo(this.getAFlowNode(), _, value, cfgorigin) and origin.getAFlowNode() = cfgorigin and if exists(value.getSource()) then obj = value.getSource() diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll index 2cd0b355ab9b..946477292d57 100755 --- a/python/ql/src/semmle/python/Flow.qll +++ b/python/ql/src/semmle/python/Flow.qll @@ -224,7 +224,7 @@ class ControlFlowNode extends @py_flow_node { /** The value and origin that this ControlFlowNode points-to, given the context. */ predicate pointsTo(Context context, Value value, ControlFlowNode origin) { - PointsTo2::points_to(this, context, value, origin) + PointsTo2::pointsTo(this, context, value, origin) } /** Gets what this flow node might "refer-to". Performs a combination of localized (intra-procedural) points-to @@ -240,7 +240,7 @@ class ControlFlowNode extends @py_flow_node { */ predicate refersTo(Context context, Object obj, ClassObject cls, ControlFlowNode origin) { exists(Value value | - PointsTo2::points_to(this, context, value, origin) and + PointsTo2::pointsTo(this, context, value, origin) and cls = value.getClass().getSource() | if exists(value.getSource().(Object)) then obj = value.getSource() @@ -255,7 +255,7 @@ class ControlFlowNode extends @py_flow_node { */ predicate refersTo(Object obj, ControlFlowNode origin) { exists(Value value | - PointsTo2::points_to(this, _, value, origin) | + PointsTo2::pointsTo(this, _, value, origin) | if exists(value.getSource().(Object)) then obj = value.getSource() else diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index cf64ebee001a..ad2b2baaba57 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -81,7 +81,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti func = this.getScope() and callee.appliesToScope(func) | rval = func.getAReturnValueFlowNode() and - PointsTo2::points_to(rval, callee, obj, origin) + PointsTo2::pointsTo(rval, callee, obj, origin) or exists(Return ret | ret.getScope() = func and diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 1c5a88f51d60..c6dc290d9d35 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -236,7 +236,7 @@ class TypeInternal extends ClassObjectInternal, TType { exists(CallNode call, PointsToContext caller, ObjectInternal instance | callee.fromCall(call, caller) | count(call.getAnArg()) = 1 and - PointsTo2::points_to(call.getArg(0), caller, instance, origin) and + PointsTo2::pointsTo(call.getArg(0), caller, instance, origin) and obj = instance.getClass() ) } diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index f339ce4705dc..e1899ab2162f 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -204,7 +204,7 @@ class FloatObjectInternal extends ConstantObjectInternal, TFloat { } override Builtin getBuiltin() { - result.getName().toFloat() = this.floatValue() + result.floatValue() = this.floatValue() } private float floatValue() { diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 695f1d2b95ce..54516995727b 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -90,7 +90,7 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { this = instance and descriptor.isDescriptor() = true and exists(AttrNode attr | - PointsTo2::points_to(attr.getObject(name), _, instance, _) and + PointsTo2::pointsTo(attr.getObject(name), _, instance, _) and this.getClass().attribute(name, descriptor, _) ) } @@ -200,7 +200,7 @@ class SelfInstanceInternal extends TSelfInstance, ObjectInternal { descriptor.isDescriptor() = true and this = instance and exists(AttrNode attr | - PointsTo2::points_to(attr.getObject(name), _, this, _) and + PointsTo2::pointsTo(attr.getObject(name), _, this, _) and this.getClass().attribute(name, descriptor, _) ) } @@ -291,7 +291,7 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { descriptor.isDescriptor() = true and this = instance and exists(AttrNode attr | - PointsTo2::points_to(attr.getObject(name), _, this, _) and + PointsTo2::pointsTo(attr.getObject(name), _, this, _) and this.getClass().attribute(name, descriptor, _) ) } @@ -366,7 +366,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal { override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { descriptor.isDescriptor() = true and exists(AttrNode attr | - PointsTo2::points_to(attr.getObject(name), _, this, _) and + PointsTo2::pointsTo(attr.getObject(name), _, this, _) and instance = this.getSelf() and Types::declaredAttribute(this.getMro().findDeclaringClass(name), name, descriptor, _) ) diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index b80644811fd8..d169f1084286 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -161,7 +161,7 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { // exists(EssaVariable var, Context context | // isModuleStateVariable(var) and var.getAUse() = init.getANormalExit() and // context.isImport() and - // SSA::ssa_variable_named_attribute_points_to(var, context, name, undefinedVariable(), _, origin) and + // SSA::ssa_variable_named_attribute_pointsTo(var, context, name, undefinedVariable(), _, origin) and // value = this.submodule(name) // ) //) @@ -233,7 +233,7 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { exists(EssaVariable var, ControlFlowNode exit, PointsToContext imp | exit = this.getSourceModule().getANormalExit() and var.getAUse() = exit and var.getSourceVariable().getName() = name and - PointsTo2::ssa_variable_points_to(var, imp, value, origin) and + PointsTo2::ssa_variable_pointsTo(var, imp, value, origin) and imp.isImport() and value != ObjectInternal::undefined() ) @@ -242,7 +242,7 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { //not exists(EssaVariable var | var.getAUse() = m.getANormalExit() and var.getSourceVariable().getName() = name) and //exists(EssaVariable var, PointsToContext imp | // var.getAUse() = m.getANormalExit() and isModuleStateVariable(var) | - // PointsTo2::ssa_variable_named_attribute_points_to(var, imp, name, obj, origin) and + // PointsTo2::ssa_variable_named_attribute_pointsTo(var, imp, name, obj, origin) and // imp.isImport() and obj != ObjectInternal::undefined() //) } diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 207e5884b1ad..f8b8ee6e085d 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -19,11 +19,11 @@ class Value extends TObject { } ControlFlowNode getAReference() { - PointsTo2::points_to(result, _, this, _) + PointsTo2::pointsTo(result, _, this, _) } predicate pointsTo(ControlFlowNode node, PointsToContext context, ControlFlowNode origin) { - PointsTo2::points_to(node, context, this, origin) + PointsTo2::pointsTo(node, context, this, origin) } Value getClass() { @@ -31,10 +31,10 @@ class Value extends TObject { } CallNode getACall() { - PointsTo2::points_to(result.getFunction(), _, this, _) + PointsTo2::pointsTo(result.getFunction(), _, this, _) or exists(BoundMethodObjectInternal bm | - PointsTo2::points_to(result.getFunction(), _, bm, _) and + PointsTo2::pointsTo(result.getFunction(), _, bm, _) and bm.getFunction() = this ) } diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index 3f022ad051a4..2504dfd60ec4 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -125,7 +125,7 @@ class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal { override ObjectInternal getItem(int n) { exists(TupleNode t, PointsToContext context | this = TPythonTuple(t, context) and - PointsTo2::points_to(t.getElement(n), context, result, _) + PointsTo2::pointsTo(t.getElement(n), context, result, _) ) } diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index cce2ebce0cc4..37af03f3cac0 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -83,7 +83,7 @@ newtype TObject = } or TSpecificInstance(ControlFlowNode instantiation, ClassObjectInternal cls, PointsToContext context) { - PointsTo2::points_to(instantiation.(CallNode).getFunction(), context, cls, _) and + PointsTo2::pointsTo(instantiation.(CallNode).getFunction(), context, cls, _) and cls.isSpecial() = false or literal_instantiation(instantiation, cls, context) @@ -128,13 +128,13 @@ private predicate is_power_2(int n) { } predicate static_method(CallNode instantiation, CallableObjectInternal function, PointsToContext context) { - PointsTo2::points_to(instantiation.getFunction(), context, ObjectInternal::builtin("staticmethod"), _) and - PointsTo2::points_to(instantiation.getArg(0), context, function, _) + PointsTo2::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("staticmethod"), _) and + PointsTo2::pointsTo(instantiation.getArg(0), context, function, _) } predicate class_method(CallNode instantiation, CallableObjectInternal function, PointsToContext context) { - PointsTo2::points_to(instantiation.getFunction(), context, ObjectInternal::builtin("classmethod"), _) and - PointsTo2::points_to(instantiation.getArg(0), context, function, _) + PointsTo2::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("classmethod"), _) and + PointsTo2::pointsTo(instantiation.getArg(0), context, function, _) } predicate literal_instantiation(ControlFlowNode n, ClassObjectInternal cls, PointsToContext context) { @@ -149,10 +149,10 @@ predicate literal_instantiation(ControlFlowNode n, ClassObjectInternal cls, Poin } predicate super_instantiation(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext context) { - PointsTo2::points_to(instantiation.getFunction(), context, ObjectInternal::builtin("super"), _) and + PointsTo2::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("super"), _) and ( - PointsTo2::points_to(instantiation.getArg(0), context, startclass, _) and - PointsTo2::points_to(instantiation.getArg(1), context, self, _) + PointsTo2::pointsTo(instantiation.getArg(0), context, startclass, _) and + PointsTo2::pointsTo(instantiation.getArg(1), context, self, _) or major_version() = 3 and not exists(instantiation.getArg(0)) and @@ -161,7 +161,7 @@ predicate super_instantiation(CallNode instantiation, ObjectInternal self, Class /* Implicit class argument is lexically enclosing scope */ func.getScope() = startclass.(PythonClassObjectInternal).getScope() and /* Implicit 'self' is the 0th parameter */ - PointsTo2::points_to(func.getArg(0).asName().getAFlowNode(), context, self, _) + PointsTo2::pointsTo(func.getArg(0).asName().getAFlowNode(), context, self, _) ) ) } @@ -192,7 +192,7 @@ predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableOb /** Helper for method_binding */ pragma [noinline] predicate receiver(AttrNode instantiation, PointsToContext context, ObjectInternal obj, string name) { - PointsTo2::points_to(instantiation.getObject(name), context, obj, _) + PointsTo2::pointsTo(instantiation.getObject(name), context, obj, _) } /** Helper self parameters: `def meth(self, ...): ...`. */ @@ -247,7 +247,7 @@ private predicate neither_class_nor_static_method(Function f) { exists(ControlFlowNode deco | deco = f.getADecorator().getAFlowNode() | exists(ObjectInternal o | - PointsTo2::points_to(deco, _, o, _) | + PointsTo2::pointsTo(deco, _, o, _) | o != ObjectInternal::staticMethod() and o != ObjectInternal::classMethod() ) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo2.qll index 79e2a2eaa8e8..6eca4d4b1a03 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo2.qll @@ -96,34 +96,34 @@ module CfgOrigin { cached module PointsTo2 { /** INTERNAL -- Use `f.refersTo(value, origin)` instead. */ - cached predicate points_to(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - points_to_candidate(f, context, value, origin) and + cached predicate pointsTo(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + pointsTo_candidate(f, context, value, origin) and reachableBlock(f.getBasicBlock(), context) } - private predicate points_to_candidate(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - use_points_to(f, context, value, origin) + private predicate pointsTo_candidate(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + use_pointsTo(f, context, value, origin) or /* Not necessary, but for backwards compatibility */ - def_points_to(f, context, value, origin) + def_pointsTo(f, context, value, origin) or - attribute_load_points_to(f, context, value, origin) + attribute_load_pointsTo(f, context, value, origin) or - subscript_points_to(f, context, value, origin) + subscript_pointsTo(f, context, value, origin) or - binary_expr_points_to(f, context, value, origin) + binary_expr_pointsTo(f, context, value, origin) or - origin = f and compare_expr_points_to(f, context, value) + origin = f and compare_expr_pointsTo(f, context, value) or - origin = f and unary_points_to(f, context, value) + origin = f and unary_pointsTo(f, context, value) or origin = f and value.introduced(f, context) or - InterModulePointsTo::import_points_to(f, context, value, origin) + InterModulePointsTo::import_pointsTo(f, context, value, origin) or - InterModulePointsTo::from_import_points_to(f, context, value, origin) + InterModulePointsTo::from_import_pointsTo(f, context, value, origin) or - InterProceduralPointsTo::call_points_to(f, context, value, origin) + InterProceduralPointsTo::call_pointsTo(f, context, value, origin) // To do... More stuff here :) // or // f.(CustomPointsToFact).pointsTo(context, value, origin) @@ -131,18 +131,18 @@ cached module PointsTo2 { /** Holds if the attribute `name` is required for `obj` */ cached predicate attributeRequired(ObjectInternal obj, string name) { - points_to(any(AttrNode a).getObject(name), _, obj, _) + pointsTo(any(AttrNode a).getObject(name), _, obj, _) or exists(CallNode call, PointsToContext ctx, StringObjectInternal nameobj | - points_to(call.getFunction(), ctx, ObjectInternal::builtin("getattr"), _) and - points_to(call.getArg(0), ctx, obj, _) and - points_to(call.getArg(1), ctx, nameobj, _) and + pointsTo(call.getFunction(), ctx, ObjectInternal::builtin("getattr"), _) and + pointsTo(call.getArg(0), ctx, obj, _) and + pointsTo(call.getArg(1), ctx, nameobj, _) and nameobj.strValue() = name ) } cached CallNode get_a_call(ObjectInternal func, PointsToContext context) { - points_to(result.getFunction(), context, func, _) + pointsTo(result.getFunction(), context, func, _) } /* Holds if BasicBlock `b` is reachable, given the context `context`. */ @@ -169,7 +169,7 @@ cached module PointsTo2 { private predicate allowsFlow(ConditionBlock guard, BasicBlock b, PointsToContext context) { exists(ObjectInternal value, boolean sense, ControlFlowNode test | test = guard.getLastNode() and - points_to(test, context, value, _) and + pointsTo(test, context, value, _) and sense = value.booleanValue() and guard.controls(b, sense) ) @@ -181,7 +181,7 @@ cached module PointsTo2 { cached predicate controlledReachableEdge(BasicBlock pred, BasicBlock succ, PointsToContext context) { exists(ConditionBlock guard, ObjectInternal value, boolean sense, ControlFlowNode test | test = guard.getLastNode() and - points_to(test, context, value, _) and + pointsTo(test, context, value, _) and sense = value.booleanValue() and guard.controlsEdge(pred, succ, sense) ) @@ -189,63 +189,63 @@ cached module PointsTo2 { /** Gets an object pointed to by a use (of a variable). */ pragma [noinline] - private predicate use_points_to(NameNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate use_pointsTo(NameNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(CfgOrigin origin_or_obj | value != ObjectInternal::undefined() and - use_points_to_maybe_origin(f, context, value, origin_or_obj) | + use_pointsTo_maybe_origin(f, context, value, origin_or_obj) | origin = origin_or_obj.asCfgNodeOrHere(f) ) } /** Gets an object pointed to by the definition of an ESSA variable. */ pragma [noinline] - private predicate def_points_to(DefinitionNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - points_to(f.getValue(), context, value, origin) + private predicate def_pointsTo(DefinitionNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + pointsTo(f.getValue(), context, value, origin) } pragma [noinline] - private predicate use_points_to_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { - ssa_variable_points_to(fast_local_variable(f), context, value, origin_or_obj) + private predicate use_pointsTo_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { + ssa_variable_pointsTo(fast_local_variable(f), context, value, origin_or_obj) or - name_lookup_points_to_maybe_origin(f, context, value, origin_or_obj) + name_lookup_pointsTo_maybe_origin(f, context, value, origin_or_obj) or not exists(fast_local_variable(f)) and not exists(name_local_variable(f)) and - global_lookup_points_to_maybe_origin(f, context, value, origin_or_obj) + global_lookup_pointsTo_maybe_origin(f, context, value, origin_or_obj) } /** Holds if `var` refers to `(value, origin)` given the context `context`. */ pragma [noinline] - cached predicate ssa_variable_points_to(EssaVariable var, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - ssa_definition_points_to(var.getDefinition(), context, value, origin) + cached predicate ssa_variable_pointsTo(EssaVariable var, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + ssa_definition_pointsTo(var.getDefinition(), context, value, origin) } pragma [noinline] - private predicate name_lookup_points_to_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { + private predicate name_lookup_pointsTo_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { exists(EssaVariable var | var = name_local_variable(f) | - ssa_variable_points_to(var, context, value, origin_or_obj) + ssa_variable_pointsTo(var, context, value, origin_or_obj) ) or local_variable_undefined(f, context) and - global_lookup_points_to_maybe_origin(f, context, value, origin_or_obj) + global_lookup_pointsTo_maybe_origin(f, context, value, origin_or_obj) } pragma [noinline] private predicate local_variable_undefined(NameNode f, PointsToContext context) { - ssa_variable_points_to(name_local_variable(f), context, ObjectInternal::undefined(), _) + ssa_variable_pointsTo(name_local_variable(f), context, ObjectInternal::undefined(), _) } pragma [noinline] - private predicate global_lookup_points_to_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { - ssa_variable_points_to(global_variable(f), context, value, origin_or_obj) + private predicate global_lookup_pointsTo_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { + ssa_variable_pointsTo(global_variable(f), context, value, origin_or_obj) or exists(ControlFlowNode origin | origin_or_obj = CfgOrigin::fromCfgNode(origin) | - ssa_variable_points_to(global_variable(f), context, ObjectInternal::undefined(), _) and - potential_builtin_points_to(f, value, origin) + ssa_variable_pointsTo(global_variable(f), context, ObjectInternal::undefined(), _) and + potential_builtin_pointsTo(f, value, origin) or not exists(global_variable(f)) and context.appliesToScope(f.getScope()) and - potential_builtin_points_to(f, value, origin) + potential_builtin_pointsTo(f, value, origin) ) } @@ -272,10 +272,10 @@ cached module PointsTo2 { /** Holds if `f` is an attribute `x.attr` and points to `(value, cls, origin)`. */ pragma [noinline] - private predicate attribute_load_points_to(AttrNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate attribute_load_pointsTo(AttrNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { f.isLoad() and exists(ObjectInternal object, string name, CfgOrigin orig | - points_to(f.getObject(name), context, object, _) | + pointsTo(f.getObject(name), context, object, _) | object.attribute(name, value, orig) and origin = orig.fix(f) or @@ -285,83 +285,83 @@ cached module PointsTo2 { // TO DO -- Support CustomPointsToAttribute //or //exists(CustomPointsToAttribute object, string name | - // points_to(f.getObject(name), context, object, _, _) and + // pointsTo(f.getObject(name), context, object, _, _) and // object.attributePointsTo(name, value, cls, origin) //) } /** Holds if the ESSA definition `def` refers to `(value, origin)` given the context `context`. */ - private predicate ssa_definition_points_to(EssaDefinition def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - ssa_phi_points_to(def, context, value, origin) + private predicate ssa_definition_pointsTo(EssaDefinition def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + ssa_phi_pointsTo(def, context, value, origin) or exists(ControlFlowNode orig | - ssa_node_definition_points_to(def, context, value, orig) and + ssa_node_definition_pointsTo(def, context, value, orig) and origin = CfgOrigin::fromCfgNode(orig) ) or - ssa_filter_definition_points_to(def, context, value, origin) + ssa_filter_definition_pointsTo(def, context, value, origin) or - ssa_node_refinement_points_to(def, context, value, origin) + ssa_node_refinement_pointsTo(def, context, value, origin) } pragma [noinline] - private predicate ssa_node_definition_points_to(EssaNodeDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate ssa_node_definition_pointsTo(EssaNodeDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { reachableBlock(def.getDefiningNode().getBasicBlock(), _) and - ssa_node_definition_points_to_unpruned(def, context, value, origin) + ssa_node_definition_pointsTo_unpruned(def, context, value, origin) } pragma [nomagic] - private predicate ssa_node_definition_points_to_unpruned(EssaNodeDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - InterProceduralPointsTo::parameter_points_to(def, context, value, origin) + private predicate ssa_node_definition_pointsTo_unpruned(EssaNodeDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + InterProceduralPointsTo::parameter_pointsTo(def, context, value, origin) or - assignment_points_to(def, context, value, origin) + assignment_pointsTo(def, context, value, origin) //// TO DO... or - self_parameter_points_to(def, context, value, origin) + self_parameter_pointsTo(def, context, value, origin) or - delete_points_to(def, context, value, origin) + delete_pointsTo(def, context, value, origin) or - module_name_points_to(def, context, value, origin) + module_name_pointsTo(def, context, value, origin) or - scope_entry_points_to(def, context, value, origin) + scope_entry_pointsTo(def, context, value, origin) or - InterModulePointsTo::implicit_submodule_points_to(def, context, value, origin) + InterModulePointsTo::implicit_submodule_pointsTo(def, context, value, origin) // or - // iteration_definition_points_to(def, context, value, origin) + // iteration_definition_pointsTo(def, context, value, origin) /* * No points-to for non-local function entry definitions yet. */ } pragma [noinline] - private predicate ssa_node_refinement_points_to(EssaNodeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - //method_callsite_points_to(def, context, value, origin) + private predicate ssa_node_refinement_pointsTo(EssaNodeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + //method_callsite_pointsTo(def, context, value, origin) //or - InterModulePointsTo::import_star_points_to(def, context, value, origin) + InterModulePointsTo::import_star_pointsTo(def, context, value, origin) or - //attribute_assignment_points_to(def, context, value, origin) + //attribute_assignment_pointsTo(def, context, value, origin) //or - InterProceduralPointsTo::callsite_points_to(def, context, value, origin) + InterProceduralPointsTo::callsite_pointsTo(def, context, value, origin) or - argument_points_to(def, context, value, origin) + argument_pointsTo(def, context, value, origin) //or - //attribute_delete_points_to(def, context, value, origin) + //attribute_delete_pointsTo(def, context, value, origin) or - uni_edged_phi_points_to(def, context, value, origin) + uni_edged_phi_pointsTo(def, context, value, origin) } /** Ignore the effects of calls on their arguments. PointsTo is an approximation, but attempting to improve accuracy would be very expensive for very little gain. */ - private predicate argument_points_to(ArgumentRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - ssa_variable_points_to(def.getInput(), context, value, origin) + private predicate argument_pointsTo(ArgumentRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + ssa_variable_pointsTo(def.getInput(), context, value, origin) } - private predicate self_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + private predicate self_parameter_pointsTo(ParameterDefinition def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { origin = CfgOrigin::fromCfgNode(def.getDefiningNode()) and value.(SelfInstanceInternal).parameterAndContext(def, context) } /** Holds if ESSA edge refinement, `def`, refers to `(value, cls, origin)`. */ - private predicate ssa_filter_definition_points_to(PyEdgeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + private predicate ssa_filter_definition_pointsTo(PyEdgeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(ControlFlowNode test, ControlFlowNode use | refinement_test(test, use, Conditionals::branchEvaluatesTo(test, use, context, value, origin.toCfgNode()), def) ) @@ -369,7 +369,7 @@ cached module PointsTo2 { /** Holds if ESSA definition, `uniphi`, refers to `(value, origin)`. */ pragma [noinline] - private predicate uni_edged_phi_points_to(SingleSuccessorGuard uniphi, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + private predicate uni_edged_phi_pointsTo(SingleSuccessorGuard uniphi, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(ControlFlowNode test, ControlFlowNode use | /* Because calls such as `len` may create a new variable, we need to go via the source variable * That is perfectly safe as we are only dealing with calls that do not mutate their arguments. @@ -382,19 +382,19 @@ cached module PointsTo2 { /** Points-to for normal assignments `def = ...`. */ pragma [noinline] - private predicate assignment_points_to(AssignmentDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - points_to(def.getValue(), context, value, origin) + private predicate assignment_pointsTo(AssignmentDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + pointsTo(def.getValue(), context, value, origin) } /** Points-to for deletion: `del name`. */ pragma [noinline] - private predicate delete_points_to(DeletionDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate delete_pointsTo(DeletionDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { value = ObjectInternal::undefined() and origin = def.getDefiningNode() and context.appliesToScope(def.getScope()) } /** Implicit "definition" of `__name__` at the start of a module. */ pragma [noinline] - private predicate module_name_points_to(ScopeEntryDefinition def, PointsToContext context, StringObjectInternal value, ControlFlowNode origin) { + private predicate module_name_pointsTo(ScopeEntryDefinition def, PointsToContext context, StringObjectInternal value, ControlFlowNode origin) { def.getVariable().getName() = "__name__" and exists(Module m | m = def.getScope() @@ -418,26 +418,26 @@ cached module PointsTo2 { /** Holds if the phi-function `phi` refers to `(value, origin)` given the context `context`. */ pragma [nomagic] - private predicate ssa_phi_points_to(PhiFunction phi, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + private predicate ssa_phi_pointsTo(PhiFunction phi, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(EssaVariable input, BasicBlock pred | input = phi.getInput(pred) and - ssa_variable_points_to(input, context, value, origin) + ssa_variable_pointsTo(input, context, value, origin) | controlledReachableEdge(pred, phi.getBasicBlock(), context) or not exists(ConditionBlock guard | guard.controlsEdge(pred, phi.getBasicBlock(), _)) ) or - ssa_variable_points_to(phi.getShortCircuitInput(), context, value, origin) + ssa_variable_pointsTo(phi.getShortCircuitInput(), context, value, origin) } /** Points-to for implicit variable declarations at scope-entry. */ pragma [noinline] - private predicate scope_entry_points_to(ScopeEntryDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate scope_entry_pointsTo(ScopeEntryDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { /* Transfer from another scope */ exists(EssaVariable var, PointsToContext outer, CfgOrigin orig | InterProceduralPointsTo::scope_entry_value_transfer(var, outer, def, context) and - ssa_variable_points_to(var, outer, value, orig) and + ssa_variable_pointsTo(var, outer, value, orig) and origin = orig.asCfgNodeOrHere(def.getDefiningNode()) ) or @@ -462,32 +462,32 @@ cached module PointsTo2 { ) } - private predicate subscript_points_to(SubscriptNode sub, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - points_to(sub.getValue(), context, ObjectInternal::unknown(), _) and + private predicate subscript_pointsTo(SubscriptNode sub, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + pointsTo(sub.getValue(), context, ObjectInternal::unknown(), _) and value = ObjectInternal::unknown() and origin = sub } /** Track bitwise expressions so we can handle integer flags and enums. * Tracking too many binary expressions is likely to kill performance. */ - private predicate binary_expr_points_to(BinaryExprNode b, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate binary_expr_pointsTo(BinaryExprNode b, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { // TO DO... // Track some integer values through `|` and the types of some objects none() } pragma [noinline] - private predicate compare_expr_points_to(CompareNode cmp, PointsToContext context, ObjectInternal value) { + private predicate compare_expr_pointsTo(CompareNode cmp, PointsToContext context, ObjectInternal value) { value = ObjectInternal::bool(Conditionals::comparisonEvaluatesTo(cmp, _, context, _, _)) // or // value = version_tuple_compare(cmp, context) } pragma [noinline] - private predicate unary_points_to(UnaryExprNode f, PointsToContext context, ObjectInternal value) { + private predicate unary_pointsTo(UnaryExprNode f, PointsToContext context, ObjectInternal value) { exists(Unaryop op, ObjectInternal operand | op = f.getNode().getOp() and - points_to(f.getOperand(), context, operand, _) + pointsTo(f.getOperand(), context, operand, _) | op instanceof Not and value = ObjectInternal::bool(operand.booleanValue().booleanNot()) or @@ -502,7 +502,7 @@ cached module PointsTo2 { module InterModulePointsTo { pragma [noinline] - predicate import_points_to(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + predicate import_pointsTo(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(string name, ImportExpr i | i.getAFlowNode() = f and i.getImportedModuleName() = name and module_imported_as(value, name) and @@ -511,23 +511,23 @@ module InterModulePointsTo { ) } - predicate from_import_points_to(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - from_self_import_points_to(f, context, value, origin) + predicate from_import_pointsTo(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + from_self_import_pointsTo(f, context, value, origin) or - from_other_import_points_to(f, context, value, origin) + from_other_import_pointsTo(f, context, value, origin) } pragma [noinline] - predicate from_self_import_points_to(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + predicate from_self_import_pointsTo(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(EssaVariable var, CfgOrigin orig | var = ssa_variable_for_module_attribute(f, context) and - PointsTo2::ssa_variable_points_to(var, context, value, orig) and + PointsTo2::ssa_variable_pointsTo(var, context, value, orig) and origin = orig.asCfgNodeOrHere(f) ) } pragma [noinline] - predicate from_other_import_points_to(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + predicate from_other_import_pointsTo(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(string name, ModuleObjectInternal mod, CfgOrigin orig | from_import_imports(f, context, mod, name) and (mod.getSourceModule() != f.getEnclosingModule() or mod.isBuiltin()) and @@ -538,20 +538,20 @@ module InterModulePointsTo { //not exists(EssaVariable var | var.getSourceVariable().getName() = name and var.getAUse() = f) and //exists(EssaVariable dollar | // isModuleStateVariable(dollar) and dollar.getAUse() = f and - // SSA::ssa_variable_named_attribute_points_to(dollar, context, name, value, orig) + // SSA::ssa_variable_named_attribute_pointsTo(dollar, context, name, value, orig) //) ) } private predicate from_import_imports(ImportMemberNode f, PointsToContext context, ModuleObjectInternal mod, string name) { - PointsTo2::points_to(f.getModule(name), context, mod, _) + PointsTo2::pointsTo(f.getModule(name), context, mod, _) } pragma [noinline] private EssaVariable ssa_variable_for_module_attribute(ImportMemberNode f, PointsToContext context) { exists(string name, ModuleObjectInternal mod, Module m | mod.getSourceModule() = m and m = f.getEnclosingModule() and m = result.getScope() and - PointsTo2::points_to(f.getModule(name), context, mod, _) and + PointsTo2::pointsTo(f.getModule(name), context, mod, _) and result.getSourceVariable().getName() = name and result.getAUse() = f ) } @@ -566,11 +566,11 @@ module InterModulePointsTo { /* Use previous points-to here to avoid slowing down the recursion too much */ exists(SubscriptNode sub | sub.getValue() = sys_modules_flow and - PointsTo2::points_to(sys_modules_flow, _, ObjectInternal::sysModules(), _) and + PointsTo2::pointsTo(sys_modules_flow, _, ObjectInternal::sysModules(), _) and sub.getIndex() = n and n.getNode().(StrConst).getText() = name and sub.(DefinitionNode).getValue() = mod and - PointsTo2::points_to(mod, _, m, _) + PointsTo2::pointsTo(mod, _, m, _) ) ) } @@ -580,7 +580,7 @@ module InterModulePointsTo { * PointsTo isn't exactly how the interpreter works, but is the best approximation we can manage statically. */ pragma [noinline] - predicate implicit_submodule_points_to(ImplicitSubModuleDefinition def, PointsToContext context, ModuleObjectInternal value, ControlFlowNode origin) { + predicate implicit_submodule_pointsTo(ImplicitSubModuleDefinition def, PointsToContext context, ModuleObjectInternal value, ControlFlowNode origin) { exists(PackageObjectInternal package | package.getSourceModule() = def.getDefiningNode().getScope() | value = package.submodule(def.getSourceVariable().getName()) and @@ -590,12 +590,12 @@ module InterModulePointsTo { } /** Points-to for `from ... import *`. */ - predicate import_star_points_to(ImportStarRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + predicate import_star_pointsTo(ImportStarRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(CfgOrigin orig | origin = orig.fix(def.getDefiningNode()) | exists(ModuleObjectInternal mod, string name | - PointsTo2::points_to(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) and + PointsTo2::pointsTo(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) and name = def.getSourceVariable().getName() | /* Attribute from imported module */ module_exports_boolean(mod, name) = true and @@ -605,7 +605,7 @@ module InterModulePointsTo { exists(EssaVariable var | /* Retain value held before import */ variable_not_redefined_by_import_star(var, context, def) and - PointsTo2::ssa_variable_points_to(var, context, value,orig) + PointsTo2::ssa_variable_pointsTo(var, context, value,orig) ) ) } @@ -615,7 +615,7 @@ module InterModulePointsTo { cached predicate variable_not_redefined_by_import_star(EssaVariable var, PointsToContext context, ImportStarRefinement def) { var = def.getInput() and exists(ModuleObjectInternal mod | - PointsTo2::points_to(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) | + PointsTo2::pointsTo(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) | module_exports_boolean(mod, var.getSourceVariable().getName()) = false or exists(Module m, string name | @@ -627,7 +627,7 @@ module InterModulePointsTo { private predicate importsByImportStar(ModuleObjectInternal mod, ModuleObjectInternal imported) { exists(ImportStarNode isn | - PointsTo2::points_to(isn.getModule(), _, imported, _) and + PointsTo2::pointsTo(isn.getModule(), _, imported, _) and isn.getScope() = mod.getSourceModule() ) or exists(ModuleObjectInternal mid | @@ -660,7 +660,7 @@ module InterModulePointsTo { else ( exists(ImportStarNode isn, ModuleObjectInternal imported | isn.getScope() = src and - PointsTo2::points_to(isn.getModule(), _, imported, _) and + PointsTo2::pointsTo(isn.getModule(), _, imported, _) and result = module_exports_boolean(imported, name) ) or @@ -686,9 +686,9 @@ module InterModulePointsTo { module InterProceduralPointsTo { pragma [noinline] - predicate call_points_to(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + predicate call_pointsTo(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(ObjectInternal func, CfgOrigin resultOrigin | - PointsTo2::points_to(f.getFunction(), context, func, _) and + PointsTo2::pointsTo(f.getFunction(), context, func, _) and origin = resultOrigin.fix(f) | exists(PointsToContext callee | @@ -702,21 +702,21 @@ module InterProceduralPointsTo { /** Points-to for parameter. `def foo(param): ...`. */ pragma [noinline] - predicate parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - positional_parameter_points_to(def, context, value, origin) + predicate parameter_pointsTo(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + positional_parameter_pointsTo(def, context, value, origin) or - named_parameter_points_to(def, context, value, origin) + named_parameter_pointsTo(def, context, value, origin) or - default_parameter_points_to(def, context, value, origin) + default_parameter_pointsTo(def, context, value, origin) or - special_parameter_points_to(def, context, value, origin) + special_parameter_pointsTo(def, context, value, origin) } - /** Helper for `parameter_points_to` */ + /** Helper for `parameter_pointsTo` */ pragma [noinline] - private predicate positional_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate positional_parameter_pointsTo(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(PointsToContext caller, ControlFlowNode arg | - PointsTo2::points_to(arg, caller, value, origin) and + PointsTo2::pointsTo(arg, caller, value, origin) and callsite_argument_transfer(arg, caller, def, context) ) or @@ -725,23 +725,23 @@ module InterProceduralPointsTo { } - /** Helper for `parameter_points_to` */ + /** Helper for `parameter_pointsTo` */ pragma [noinline] - private predicate named_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate named_parameter_pointsTo(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(CallNode call, PointsToContext caller, PythonFunctionObjectInternal func, string name | context.fromCall(call, func, caller) and def.getParameter() = func.getScope().getArgByName(name) and - PointsTo2::points_to(call.getArgByName(name), caller, value, origin) + PointsTo2::pointsTo(call.getArgByName(name), caller, value, origin) ) } - /** Helper for parameter_points_to */ - private predicate default_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - exists(PointsToContext imp | imp.isImport() | PointsTo2::points_to(def.getDefault(), imp, value, origin)) and + /** Helper for parameter_pointsTo */ + private predicate default_parameter_pointsTo(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + exists(PointsToContext imp | imp.isImport() | PointsTo2::pointsTo(def.getDefault(), imp, value, origin)) and context_for_default_value(def, context) } - /** Helper for default_parameter_points_to */ + /** Helper for default_parameter_pointsTo */ pragma [noinline] private predicate context_for_default_value(ParameterDefinition def, PointsToContext context) { context.isRuntime() @@ -756,9 +756,9 @@ module InterProceduralPointsTo { ) } - /** Helper for parameter_points_to */ + /** Helper for parameter_pointsTo */ pragma [noinline] - private predicate special_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate special_parameter_pointsTo(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { context.isRuntime() and origin = def.getDefiningNode() and exists(ControlFlowNode param | @@ -792,7 +792,7 @@ module InterProceduralPointsTo { cached predicate callsite_calls_function(CallNode call, PointsToContext caller, Function scope, PointsToContext callee, int parameter_offset) { callee.fromCall(call, caller) and exists(ObjectInternal func | - PointsTo2::points_to(call.getFunction(), caller, func, _) and + PointsTo2::pointsTo(call.getFunction(), caller, func, _) and func.calleeAndOffset(scope, parameter_offset) ) } @@ -861,7 +861,7 @@ module InterProceduralPointsTo { * Where var may be redefined in call to `foo` if `var` escapes (is global or non-local). */ pragma [noinline] - predicate callsite_points_to(CallsiteRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + predicate callsite_pointsTo(CallsiteRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(SsaSourceVariable srcvar | srcvar = def.getSourceVariable() | if srcvar instanceof EscapingAssignmentGlobalVariable then ( @@ -869,17 +869,17 @@ module InterProceduralPointsTo { exists(EssaVariable var, Function func, PointsToContext callee | callsite_calls_function(def.getCall(), context, func, callee, _) and var_at_exit(srcvar, func, var) and - PointsTo2::ssa_variable_points_to(var, callee, value, origin) + PointsTo2::ssa_variable_pointsTo(var, callee, value, origin) ) or exists(ObjectInternal callable | - PointsTo2::points_to(def.getCall().getFunction(), context, callable, _) and + PointsTo2::pointsTo(def.getCall().getFunction(), context, callable, _) and exists(callable.getBuiltin()) and - PointsTo2::ssa_variable_points_to(def.getInput(), context, value, origin) + PointsTo2::ssa_variable_pointsTo(def.getInput(), context, value, origin) ) ) else ( /* Otherwise we can assume its value (but not those of its attributes or members) has not changed. */ - PointsTo2::ssa_variable_points_to(def.getInput(), context, value, origin) + PointsTo2::ssa_variable_pointsTo(def.getInput(), context, value, origin) ) ) } @@ -896,7 +896,7 @@ module InterProceduralPointsTo { /** Gets the `value, origin` that `f` would refer to if it has not been assigned some other value */ pragma [noinline] -private predicate potential_builtin_points_to(NameNode f, ObjectInternal value, ControlFlowNode origin) { +private predicate potential_builtin_pointsTo(NameNode f, ObjectInternal value, ControlFlowNode origin) { f.isGlobal() and f.isLoad() and origin = f and ( value = ObjectInternal::builtin(f.getId()) @@ -915,7 +915,7 @@ private module Conditionals { boolean branchEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { contains_interesting_expression_within_test(expr, use) and - PointsTo2::points_to(use, context, val, origin) and + PointsTo2::pointsTo(use, context, val, origin) and expr = use and val.booleanValue() = result or @@ -943,8 +943,8 @@ private module Conditionals { exists(ControlFlowNode r, boolean sense | equality_test(expr, use, sense, r) and exists(ObjectInternal other | - PointsTo2::points_to(use, context, val, origin) and - PointsTo2::points_to(r, context, other, _) | + PointsTo2::pointsTo(use, context, val, origin) and + PointsTo2::pointsTo(r, context, other, _) | val.isComparable() = true and other.isComparable() = true and ( other = val and result = sense @@ -968,8 +968,8 @@ private module Conditionals { or inequality(expr, r, use, strict) and sense = false ) and - PointsTo2::points_to(use, context, val, origin) and - PointsTo2::points_to(r, context, other, _) + PointsTo2::pointsTo(use, context, val, origin) and + PointsTo2::pointsTo(r, context, other, _) | val.intValue() < other.intValue() and result = sense or @@ -1028,7 +1028,7 @@ cached module Types { or exists(Class pycls | pycls = cls.(PythonClassObjectInternal).getScope() | - PointsTo2::points_to(pycls.getBase(n).getAFlowNode(), _, result, _) + PointsTo2::pointsTo(pycls.getBase(n).getAFlowNode(), _, result, _) or not exists(pycls.getABase()) and n = 0 and isNewStyle(cls) and result = ObjectInternal::builtin("object") @@ -1063,7 +1063,7 @@ cached module Types { exists(EssaVariable var | name = var.getName() and var.getAUse() = cls.(PythonClassObjectInternal).getScope().getANormalExit() and - PointsTo2::ssa_variable_points_to(var, _, value, origin) + PointsTo2::ssa_variable_pointsTo(var, _, value, origin) ) } @@ -1075,7 +1075,7 @@ cached module Types { private ClassObjectInternal declaredMetaClass(PythonClassObjectInternal cls) { exists(ObjectInternal obj | - PointsTo2::ssa_variable_points_to(metaclass_var(cls.getScope()), _, obj, _) | + PointsTo2::ssa_variable_pointsTo(metaclass_var(cls.getScope()), _, obj, _) | result = obj or obj = ObjectInternal::unknown() and result = ObjectInternal::unknownClass() @@ -1083,7 +1083,7 @@ cached module Types { or exists(ControlFlowNode meta | six_add_metaclass(_, cls, meta) and - PointsTo2::points_to(meta, _, result, _) + PointsTo2::pointsTo(meta, _, result, _) ) } @@ -1098,7 +1098,7 @@ cached module Types { private boolean has_metaclass_var_metaclass(PythonClassObjectInternal cls) { exists(ObjectInternal obj | - PointsTo2::ssa_variable_points_to(metaclass_var(cls.getScope()), _, obj, _) | + PointsTo2::ssa_variable_pointsTo(metaclass_var(cls.getScope()), _, obj, _) | obj = ObjectInternal::undefined() and result = false or obj != ObjectInternal::undefined() and result = true @@ -1126,11 +1126,11 @@ cached module Types { // decorator_call.getArg(0) = decorated and // decorator = decorator_call.getFunction() and // decorator.getArg(0) = metaclass | - // PointsTo2::points_to(decorator.getFunction(), _, six_add_metaclass_function(), _) + // PointsTo2::pointsTo(decorator.getFunction(), _, six_add_metaclass_function(), _) // or // exists(ModuleObjectInternal six | // six.getName() = "six" and - // PointsTo2::points_to(decorator.getFunction().(AttrNode).getObject("add_metaclass"), _, six, _) + // PointsTo2::pointsTo(decorator.getFunction().(AttrNode).getObject("add_metaclass"), _, six, _) // ) //) } From f7edbcc6d9337975e2e710a6065938ae2deac4ac Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 27 Mar 2019 14:31:49 +0000 Subject: [PATCH 021/128] Python points-to: Clean up interface, and deprecate old interface. --- python/ql/src/semmle/python/Exprs.qll | 6 +- python/ql/src/semmle/python/Flow.qll | 8 +- .../src/semmle/python/objects/Callables.qll | 30 +- .../ql/src/semmle/python/objects/Classes.qll | 6 +- .../src/semmle/python/objects/Constants.qll | 2 +- .../src/semmle/python/objects/Instances.qll | 18 +- .../ql/src/semmle/python/objects/Modules.qll | 6 +- .../src/semmle/python/objects/ObjectAPI.qll | 19 +- .../semmle/python/objects/ObjectInternal.qll | 2 +- .../src/semmle/python/objects/Sequences.qll | 4 +- .../ql/src/semmle/python/objects/TObject.qll | 24 +- python/ql/src/semmle/python/pointsto/MRO2.qll | 2 +- .../pointsto/{PointsTo2.qll => PointsTo.qll} | 304 ++++++++++-------- .../python/pointsto/PointsToContext.qll | 6 +- .../src/semmle/python/types/ClassObject.qll | 2 +- .../src/semmle/python/types/Descriptors.qll | 1 - .../semmle/python/types/FunctionObject.qll | 4 +- .../src/semmle/python/types/ModuleObject.qll | 2 +- 18 files changed, 257 insertions(+), 189 deletions(-) rename python/ql/src/semmle/python/pointsto/{PointsTo2.qll => PointsTo.qll} (76%) diff --git a/python/ql/src/semmle/python/Exprs.qll b/python/ql/src/semmle/python/Exprs.qll index 5c36b0d8889d..2dc201f7bf42 100644 --- a/python/ql/src/semmle/python/Exprs.qll +++ b/python/ql/src/semmle/python/Exprs.qll @@ -1,5 +1,5 @@ import python -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo /** An expression */ class Expr extends Expr_, AstNode { @@ -86,7 +86,7 @@ class Expr extends Expr_, AstNode { */ predicate refersTo(Context context, Object obj, ClassObject cls, AstNode origin) { exists(Value value, ControlFlowNode cfgorigin | - PointsTo2::pointsTo(this.getAFlowNode(), context, value, cfgorigin) and + PointsTo::pointsTo(this.getAFlowNode(), context, value, cfgorigin) and origin.getAFlowNode() = cfgorigin and cls = value.getClass().getSource() | if exists(value.getSource()) then @@ -102,7 +102,7 @@ class Expr extends Expr_, AstNode { */ predicate refersTo(Object obj, AstNode origin) { exists(Value value, ControlFlowNode cfgorigin | - PointsTo2::pointsTo(this.getAFlowNode(), _, value, cfgorigin) and + PointsTo::pointsTo(this.getAFlowNode(), _, value, cfgorigin) and origin.getAFlowNode() = cfgorigin and if exists(value.getSource()) then obj = value.getSource() diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll index 946477292d57..94d4942620a3 100755 --- a/python/ql/src/semmle/python/Flow.qll +++ b/python/ql/src/semmle/python/Flow.qll @@ -1,6 +1,6 @@ import python import semmle.python.flow.NameNode -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo /* Note about matching parent and child nodes and CFG splitting: @@ -224,7 +224,7 @@ class ControlFlowNode extends @py_flow_node { /** The value and origin that this ControlFlowNode points-to, given the context. */ predicate pointsTo(Context context, Value value, ControlFlowNode origin) { - PointsTo2::pointsTo(this, context, value, origin) + PointsTo::pointsTo(this, context, value, origin) } /** Gets what this flow node might "refer-to". Performs a combination of localized (intra-procedural) points-to @@ -240,7 +240,7 @@ class ControlFlowNode extends @py_flow_node { */ predicate refersTo(Context context, Object obj, ClassObject cls, ControlFlowNode origin) { exists(Value value | - PointsTo2::pointsTo(this, context, value, origin) and + PointsTo::pointsTo(this, context, value, origin) and cls = value.getClass().getSource() | if exists(value.getSource().(Object)) then obj = value.getSource() @@ -255,7 +255,7 @@ class ControlFlowNode extends @py_flow_node { */ predicate refersTo(Object obj, ControlFlowNode origin) { exists(Value value | - PointsTo2::pointsTo(this, _, value, origin) | + PointsTo::pointsTo(this, _, value, origin) | if exists(value.getSource().(Object)) then obj = value.getSource() else diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index ad2b2baaba57..7ac02a635048 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -3,7 +3,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext private import semmle.python.pointsto.MRO2 private import semmle.python.types.Builtins @@ -42,6 +42,10 @@ abstract class CallableObjectInternal extends ObjectInternal { override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + abstract CallNode getACall(PointsToContext ctx); + + CallNode getACall() { result = this.getACall(_) } + } @@ -81,11 +85,11 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti func = this.getScope() and callee.appliesToScope(func) | rval = func.getAReturnValueFlowNode() and - PointsTo2::pointsTo(rval, callee, obj, origin) + PointsToInternal::pointsTo(rval, callee, obj, origin) or exists(Return ret | ret.getScope() = func and - PointsTo2::reachableBlock(ret.getAFlowNode().getBasicBlock(), callee) and + PointsToInternal::reachableBlock(ret.getAFlowNode().getBasicBlock(), callee) and not exists(ret.getValue()) and obj = ObjectInternal::none_() and origin = CfgOrigin::unknown() @@ -113,6 +117,14 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti value = TBoundMethod(instance, this) and origin = CfgOrigin::unknown() } + override CallNode getACall(PointsToContext ctx) { + PointsTo::pointsTo(result.getFunction(), ctx, this, _) + or + exists(BoundMethodObjectInternal bm | + bm.getACall() = result and this = bm.getFunction() + ) + } + } class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunctionObject { @@ -192,6 +204,10 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + override CallNode getACall(PointsToContext ctx) { + PointsTo::pointsTo(result.getFunction(), ctx, this, _) + } + } @@ -248,6 +264,10 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod value = this and origin = CfgOrigin::unknown() } + override CallNode getACall(PointsToContext ctx) { + PointsTo::pointsTo(result.getFunction(), ctx, this, _) + } + } class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { @@ -307,6 +327,10 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + override CallNode getACall(PointsToContext ctx) { + PointsTo::pointsTo(result.getFunction(), ctx, this, _) + } + } class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index c6dc290d9d35..297208171f6b 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -3,7 +3,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext private import semmle.python.pointsto.MRO2 private import semmle.python.types.Builtins @@ -39,7 +39,7 @@ abstract class ClassObjectInternal extends ObjectInternal { override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { instance = this and - PointsTo2::attributeRequired(this, name) and + PointsToInternal::attributeRequired(this, name) and this.attribute(name, descriptor, _) and descriptor.isDescriptor() = true } @@ -236,7 +236,7 @@ class TypeInternal extends ClassObjectInternal, TType { exists(CallNode call, PointsToContext caller, ObjectInternal instance | callee.fromCall(call, caller) | count(call.getAnArg()) = 1 and - PointsTo2::pointsTo(call.getArg(0), caller, instance, origin) and + PointsToInternal::pointsTo(call.getArg(0), caller, instance, origin) and obj = instance.getClass() ) } diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index e1899ab2162f..cc15bc7ab852 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -2,7 +2,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.MRO2 private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 54516995727b..c12108e364ed 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -3,7 +3,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.MRO2 private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins @@ -77,7 +77,7 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { } override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - PointsTo2::attributeRequired(this, name) and + PointsToInternal::attributeRequired(this, name) and instance_getattr(this, Types::getMro(this.getClass()), name, value, origin) } @@ -90,7 +90,7 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { this = instance and descriptor.isDescriptor() = true and exists(AttrNode attr | - PointsTo2::pointsTo(attr.getObject(name), _, instance, _) and + PointsToInternal::pointsTo(attr.getObject(name), _, instance, _) and this.getClass().attribute(name, descriptor, _) ) } @@ -186,7 +186,7 @@ class SelfInstanceInternal extends TSelfInstance, ObjectInternal { } override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - PointsTo2::attributeRequired(this, name) and + PointsToInternal::attributeRequired(this, name) and instance_getattr(this, Types::getMro(this.getClass()), name, value, origin) } @@ -200,7 +200,7 @@ class SelfInstanceInternal extends TSelfInstance, ObjectInternal { descriptor.isDescriptor() = true and this = instance and exists(AttrNode attr | - PointsTo2::pointsTo(attr.getObject(name), _, this, _) and + PointsToInternal::pointsTo(attr.getObject(name), _, this, _) and this.getClass().attribute(name, descriptor, _) ) } @@ -277,7 +277,7 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { } override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - PointsTo2::attributeRequired(this, name) and + PointsToInternal::attributeRequired(this, name) and instance_getattr(this, Types::getMro(this.getClass()), name, value, origin) } @@ -291,7 +291,7 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { descriptor.isDescriptor() = true and this = instance and exists(AttrNode attr | - PointsTo2::pointsTo(attr.getObject(name), _, this, _) and + PointsToInternal::pointsTo(attr.getObject(name), _, this, _) and this.getClass().attribute(name, descriptor, _) ) } @@ -355,7 +355,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal { override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - PointsTo2::attributeRequired(this, name) and + PointsToInternal::attributeRequired(this, name) and instance_getattr(this.getSelf(), this.getMro(), name, value, origin) } @@ -366,7 +366,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal { override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { descriptor.isDescriptor() = true and exists(AttrNode attr | - PointsTo2::pointsTo(attr.getObject(name), _, this, _) and + PointsToInternal::pointsTo(attr.getObject(name), _, this, _) and instance = this.getSelf() and Types::declaredAttribute(this.getMro().findDeclaringClass(name), name, descriptor, _) ) diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index d169f1084286..471ed3ab0429 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -2,7 +2,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.MRO2 private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins @@ -233,7 +233,7 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { exists(EssaVariable var, ControlFlowNode exit, PointsToContext imp | exit = this.getSourceModule().getANormalExit() and var.getAUse() = exit and var.getSourceVariable().getName() = name and - PointsTo2::ssa_variable_pointsTo(var, imp, value, origin) and + PointsTo::variablePointsTo(var, imp, value, origin) and imp.isImport() and value != ObjectInternal::undefined() ) @@ -242,7 +242,7 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { //not exists(EssaVariable var | var.getAUse() = m.getANormalExit() and var.getSourceVariable().getName() = name) and //exists(EssaVariable var, PointsToContext imp | // var.getAUse() = m.getANormalExit() and isModuleStateVariable(var) | - // PointsTo2::ssa_variable_named_attribute_pointsTo(var, imp, name, obj, origin) and + // PointsToInternal::ssa_variable_named_attribute_pointsTo(var, imp, name, obj, origin) and // imp.isImport() and obj != ObjectInternal::undefined() //) } diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index f8b8ee6e085d..0d60f4b5f069 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -1,7 +1,7 @@ import python private import TObject private import semmle.python.objects.ObjectInternal -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext class ObjectSource = Object; @@ -19,11 +19,11 @@ class Value extends TObject { } ControlFlowNode getAReference() { - PointsTo2::pointsTo(result, _, this, _) + PointsToInternal::pointsTo(result, _, this, _) } predicate pointsTo(ControlFlowNode node, PointsToContext context, ControlFlowNode origin) { - PointsTo2::pointsTo(node, context, this, origin) + PointsToInternal::pointsTo(node, context, this, origin) } Value getClass() { @@ -31,10 +31,19 @@ class Value extends TObject { } CallNode getACall() { - PointsTo2::pointsTo(result.getFunction(), _, this, _) + PointsToInternal::pointsTo(result.getFunction(), _, this, _) or exists(BoundMethodObjectInternal bm | - PointsTo2::pointsTo(result.getFunction(), _, bm, _) and + PointsToInternal::pointsTo(result.getFunction(), _, bm, _) and + bm.getFunction() = this + ) + } + + CallNode getACall(PointsToContext caller) { + PointsToInternal::pointsTo(result.getFunction(), caller, this, _) + or + exists(BoundMethodObjectInternal bm | + PointsToInternal::pointsTo(result.getFunction(), caller, bm, _) and bm.getFunction() = this ) } diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 2966fae9169d..38f36fcb4368 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -1,7 +1,7 @@ import python private import semmle.python.objects.TObject -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.MRO2 private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index 2504dfd60ec4..00eeed8e5213 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -6,7 +6,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext private import semmle.python.pointsto.MRO2 private import semmle.python.types.Builtins @@ -125,7 +125,7 @@ class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal { override ObjectInternal getItem(int n) { exists(TupleNode t, PointsToContext context | this = TPythonTuple(t, context) and - PointsTo2::pointsTo(t.getElement(n), context, result, _) + PointsToInternal::pointsTo(t.getElement(n), context, result, _) ) } diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 37af03f3cac0..80af6d93fbca 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -1,7 +1,7 @@ import python private import semmle.python.types.Builtins private import semmle.python.objects.ObjectInternal -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext newtype TObject = @@ -83,7 +83,7 @@ newtype TObject = } or TSpecificInstance(ControlFlowNode instantiation, ClassObjectInternal cls, PointsToContext context) { - PointsTo2::pointsTo(instantiation.(CallNode).getFunction(), context, cls, _) and + PointsToInternal::pointsTo(instantiation.(CallNode).getFunction(), context, cls, _) and cls.isSpecial() = false or literal_instantiation(instantiation, cls, context) @@ -128,13 +128,13 @@ private predicate is_power_2(int n) { } predicate static_method(CallNode instantiation, CallableObjectInternal function, PointsToContext context) { - PointsTo2::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("staticmethod"), _) and - PointsTo2::pointsTo(instantiation.getArg(0), context, function, _) + PointsToInternal::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("staticmethod"), _) and + PointsToInternal::pointsTo(instantiation.getArg(0), context, function, _) } predicate class_method(CallNode instantiation, CallableObjectInternal function, PointsToContext context) { - PointsTo2::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("classmethod"), _) and - PointsTo2::pointsTo(instantiation.getArg(0), context, function, _) + PointsToInternal::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("classmethod"), _) and + PointsToInternal::pointsTo(instantiation.getArg(0), context, function, _) } predicate literal_instantiation(ControlFlowNode n, ClassObjectInternal cls, PointsToContext context) { @@ -149,10 +149,10 @@ predicate literal_instantiation(ControlFlowNode n, ClassObjectInternal cls, Poin } predicate super_instantiation(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext context) { - PointsTo2::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("super"), _) and + PointsToInternal::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("super"), _) and ( - PointsTo2::pointsTo(instantiation.getArg(0), context, startclass, _) and - PointsTo2::pointsTo(instantiation.getArg(1), context, self, _) + PointsToInternal::pointsTo(instantiation.getArg(0), context, startclass, _) and + PointsToInternal::pointsTo(instantiation.getArg(1), context, self, _) or major_version() = 3 and not exists(instantiation.getArg(0)) and @@ -161,7 +161,7 @@ predicate super_instantiation(CallNode instantiation, ObjectInternal self, Class /* Implicit class argument is lexically enclosing scope */ func.getScope() = startclass.(PythonClassObjectInternal).getScope() and /* Implicit 'self' is the 0th parameter */ - PointsTo2::pointsTo(func.getArg(0).asName().getAFlowNode(), context, self, _) + PointsToInternal::pointsTo(func.getArg(0).asName().getAFlowNode(), context, self, _) ) ) } @@ -192,7 +192,7 @@ predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableOb /** Helper for method_binding */ pragma [noinline] predicate receiver(AttrNode instantiation, PointsToContext context, ObjectInternal obj, string name) { - PointsTo2::pointsTo(instantiation.getObject(name), context, obj, _) + PointsToInternal::pointsTo(instantiation.getObject(name), context, obj, _) } /** Helper self parameters: `def meth(self, ...): ...`. */ @@ -247,7 +247,7 @@ private predicate neither_class_nor_static_method(Function f) { exists(ControlFlowNode deco | deco = f.getADecorator().getAFlowNode() | exists(ObjectInternal o | - PointsTo2::pointsTo(deco, _, o, _) | + PointsToInternal::pointsTo(deco, _, o, _) | o != ObjectInternal::staticMethod() and o != ObjectInternal::classMethod() ) diff --git a/python/ql/src/semmle/python/pointsto/MRO2.qll b/python/ql/src/semmle/python/pointsto/MRO2.qll index 4137c0352ab7..4c336c1abdbe 100644 --- a/python/ql/src/semmle/python/pointsto/MRO2.qll +++ b/python/ql/src/semmle/python/pointsto/MRO2.qll @@ -20,7 +20,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins diff --git a/python/ql/src/semmle/python/pointsto/PointsTo2.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll similarity index 76% rename from python/ql/src/semmle/python/pointsto/PointsTo2.qll rename to python/ql/src/semmle/python/pointsto/PointsTo.qll index 6eca4d4b1a03..b3a1935e4323 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo2.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -93,37 +93,77 @@ module CfgOrigin { } -cached module PointsTo2 { +/* The API */ +module PointsTo { + + predicate pointsTo(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + PointsToInternal::pointsTo(f, context, value, origin) + } + + predicate variablePointsTo(EssaVariable var, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + PointsToInternal::variablePointsTo(var, context, value, origin) + } + + /* Backwards compatibility */ + deprecated predicate + points_to(ControlFlowNode f, PointsToContext context, Object obj, ClassObject cls, ControlFlowNode origin) { + exists(Value value | + PointsToInternal::pointsTo(f, context, value, origin) and + obj = value.getSource() and + cls = value.getClass().getSource() + ) + } + + deprecated predicate + ssa_variable_points_to(EssaVariable var, PointsToContext context, Object obj, ClassObject cls, CfgOrigin origin) { + exists(Value value | + PointsToInternal::variablePointsTo(var, context, value, origin) and + obj = value.getSource() and + cls = value.getClass().getSource() + ) + } + + deprecated + CallNode get_a_call(Object func, PointsToContext context) { + exists(Value value | + result = value.getACall(context) and + func = value.getSource() + ) + } + +} + +cached module PointsToInternal { /** INTERNAL -- Use `f.refersTo(value, origin)` instead. */ cached predicate pointsTo(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - pointsTo_candidate(f, context, value, origin) and + points_to_candidate(f, context, value, origin) and reachableBlock(f.getBasicBlock(), context) } - private predicate pointsTo_candidate(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - use_pointsTo(f, context, value, origin) + private predicate points_to_candidate(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + use_points_to(f, context, value, origin) or /* Not necessary, but for backwards compatibility */ - def_pointsTo(f, context, value, origin) + def_points_to(f, context, value, origin) or - attribute_load_pointsTo(f, context, value, origin) + attribute_load_points_to(f, context, value, origin) or - subscript_pointsTo(f, context, value, origin) + subscript_points_to(f, context, value, origin) or - binary_expr_pointsTo(f, context, value, origin) + binary_expr_points_to(f, context, value, origin) or - origin = f and compare_expr_pointsTo(f, context, value) + origin = f and compare_expr_points_to(f, context, value) or - origin = f and unary_pointsTo(f, context, value) + origin = f and unary_points_to(f, context, value) or origin = f and value.introduced(f, context) or - InterModulePointsTo::import_pointsTo(f, context, value, origin) + InterModulePointsTo::import_points_to(f, context, value, origin) or - InterModulePointsTo::from_import_pointsTo(f, context, value, origin) + InterModulePointsTo::from_import_points_to(f, context, value, origin) or - InterProceduralPointsTo::call_pointsTo(f, context, value, origin) + InterProceduralPointsTo::call_points_to(f, context, value, origin) // To do... More stuff here :) // or // f.(CustomPointsToFact).pointsTo(context, value, origin) @@ -141,10 +181,6 @@ cached module PointsTo2 { ) } - cached CallNode get_a_call(ObjectInternal func, PointsToContext context) { - pointsTo(result.getFunction(), context, func, _) - } - /* Holds if BasicBlock `b` is reachable, given the context `context`. */ cached predicate reachableBlock(BasicBlock b, PointsToContext context) { context.appliesToScope(b.getScope()) and not exists(ConditionBlock guard | guard.controls(b, _)) @@ -189,63 +225,63 @@ cached module PointsTo2 { /** Gets an object pointed to by a use (of a variable). */ pragma [noinline] - private predicate use_pointsTo(NameNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate use_points_to(NameNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(CfgOrigin origin_or_obj | value != ObjectInternal::undefined() and - use_pointsTo_maybe_origin(f, context, value, origin_or_obj) | + use_points_to_maybe_origin(f, context, value, origin_or_obj) | origin = origin_or_obj.asCfgNodeOrHere(f) ) } /** Gets an object pointed to by the definition of an ESSA variable. */ pragma [noinline] - private predicate def_pointsTo(DefinitionNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate def_points_to(DefinitionNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { pointsTo(f.getValue(), context, value, origin) } pragma [noinline] - private predicate use_pointsTo_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { - ssa_variable_pointsTo(fast_local_variable(f), context, value, origin_or_obj) + private predicate use_points_to_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { + variablePointsTo(fast_local_variable(f), context, value, origin_or_obj) or - name_lookup_pointsTo_maybe_origin(f, context, value, origin_or_obj) + name_lookup_points_to_maybe_origin(f, context, value, origin_or_obj) or not exists(fast_local_variable(f)) and not exists(name_local_variable(f)) and - global_lookup_pointsTo_maybe_origin(f, context, value, origin_or_obj) + global_lookup_points_to_maybe_origin(f, context, value, origin_or_obj) } /** Holds if `var` refers to `(value, origin)` given the context `context`. */ pragma [noinline] - cached predicate ssa_variable_pointsTo(EssaVariable var, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - ssa_definition_pointsTo(var.getDefinition(), context, value, origin) + cached predicate variablePointsTo(EssaVariable var, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + ssa_definition_points_to(var.getDefinition(), context, value, origin) } pragma [noinline] - private predicate name_lookup_pointsTo_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { + private predicate name_lookup_points_to_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { exists(EssaVariable var | var = name_local_variable(f) | - ssa_variable_pointsTo(var, context, value, origin_or_obj) + variablePointsTo(var, context, value, origin_or_obj) ) or local_variable_undefined(f, context) and - global_lookup_pointsTo_maybe_origin(f, context, value, origin_or_obj) + global_lookup_points_to_maybe_origin(f, context, value, origin_or_obj) } pragma [noinline] private predicate local_variable_undefined(NameNode f, PointsToContext context) { - ssa_variable_pointsTo(name_local_variable(f), context, ObjectInternal::undefined(), _) + variablePointsTo(name_local_variable(f), context, ObjectInternal::undefined(), _) } pragma [noinline] - private predicate global_lookup_pointsTo_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { - ssa_variable_pointsTo(global_variable(f), context, value, origin_or_obj) + private predicate global_lookup_points_to_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { + variablePointsTo(global_variable(f), context, value, origin_or_obj) or exists(ControlFlowNode origin | origin_or_obj = CfgOrigin::fromCfgNode(origin) | - ssa_variable_pointsTo(global_variable(f), context, ObjectInternal::undefined(), _) and - potential_builtin_pointsTo(f, value, origin) + variablePointsTo(global_variable(f), context, ObjectInternal::undefined(), _) and + potential_builtin_points_to(f, value, origin) or not exists(global_variable(f)) and context.appliesToScope(f.getScope()) and - potential_builtin_pointsTo(f, value, origin) + potential_builtin_points_to(f, value, origin) ) } @@ -272,7 +308,7 @@ cached module PointsTo2 { /** Holds if `f` is an attribute `x.attr` and points to `(value, cls, origin)`. */ pragma [noinline] - private predicate attribute_load_pointsTo(AttrNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate attribute_load_points_to(AttrNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { f.isLoad() and exists(ObjectInternal object, string name, CfgOrigin orig | pointsTo(f.getObject(name), context, object, _) | @@ -291,77 +327,77 @@ cached module PointsTo2 { } /** Holds if the ESSA definition `def` refers to `(value, origin)` given the context `context`. */ - private predicate ssa_definition_pointsTo(EssaDefinition def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - ssa_phi_pointsTo(def, context, value, origin) + private predicate ssa_definition_points_to(EssaDefinition def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + ssa_phi_points_to(def, context, value, origin) or exists(ControlFlowNode orig | - ssa_node_definition_pointsTo(def, context, value, orig) and + ssa_node_definition_points_to(def, context, value, orig) and origin = CfgOrigin::fromCfgNode(orig) ) or - ssa_filter_definition_pointsTo(def, context, value, origin) + ssa_filter_definition_points_to(def, context, value, origin) or - ssa_node_refinement_pointsTo(def, context, value, origin) + ssa_node_refinement_points_to(def, context, value, origin) } pragma [noinline] - private predicate ssa_node_definition_pointsTo(EssaNodeDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate ssa_node_definition_points_to(EssaNodeDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { reachableBlock(def.getDefiningNode().getBasicBlock(), _) and - ssa_node_definition_pointsTo_unpruned(def, context, value, origin) + ssa_node_definition_points_to_unpruned(def, context, value, origin) } pragma [nomagic] - private predicate ssa_node_definition_pointsTo_unpruned(EssaNodeDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - InterProceduralPointsTo::parameter_pointsTo(def, context, value, origin) + private predicate ssa_node_definition_points_to_unpruned(EssaNodeDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + InterProceduralPointsTo::parameter_points_to(def, context, value, origin) or - assignment_pointsTo(def, context, value, origin) + assignment_points_to(def, context, value, origin) //// TO DO... or - self_parameter_pointsTo(def, context, value, origin) + self_parameter_points_to(def, context, value, origin) or - delete_pointsTo(def, context, value, origin) + delete_points_to(def, context, value, origin) or - module_name_pointsTo(def, context, value, origin) + module_name_points_to(def, context, value, origin) or - scope_entry_pointsTo(def, context, value, origin) + scope_entry_points_to(def, context, value, origin) or - InterModulePointsTo::implicit_submodule_pointsTo(def, context, value, origin) + InterModulePointsTo::implicit_submodule_points_to(def, context, value, origin) // or - // iteration_definition_pointsTo(def, context, value, origin) + // iteration_definition_points_to(def, context, value, origin) /* * No points-to for non-local function entry definitions yet. */ } pragma [noinline] - private predicate ssa_node_refinement_pointsTo(EssaNodeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - //method_callsite_pointsTo(def, context, value, origin) + private predicate ssa_node_refinement_points_to(EssaNodeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + //method_callsite_points_to(def, context, value, origin) //or - InterModulePointsTo::import_star_pointsTo(def, context, value, origin) + InterModulePointsTo::import_star_points_to(def, context, value, origin) or - //attribute_assignment_pointsTo(def, context, value, origin) + //attribute_assignment_points_to(def, context, value, origin) //or - InterProceduralPointsTo::callsite_pointsTo(def, context, value, origin) + InterProceduralPointsTo::callsite_points_to(def, context, value, origin) or - argument_pointsTo(def, context, value, origin) + argument_points_to(def, context, value, origin) //or - //attribute_delete_pointsTo(def, context, value, origin) + //attribute_delete_points_to(def, context, value, origin) or - uni_edged_phi_pointsTo(def, context, value, origin) + uni_edged_phi_points_to(def, context, value, origin) } /** Ignore the effects of calls on their arguments. PointsTo is an approximation, but attempting to improve accuracy would be very expensive for very little gain. */ - private predicate argument_pointsTo(ArgumentRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - ssa_variable_pointsTo(def.getInput(), context, value, origin) + private predicate argument_points_to(ArgumentRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + variablePointsTo(def.getInput(), context, value, origin) } - private predicate self_parameter_pointsTo(ParameterDefinition def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + private predicate self_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { origin = CfgOrigin::fromCfgNode(def.getDefiningNode()) and value.(SelfInstanceInternal).parameterAndContext(def, context) } /** Holds if ESSA edge refinement, `def`, refers to `(value, cls, origin)`. */ - private predicate ssa_filter_definition_pointsTo(PyEdgeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + private predicate ssa_filter_definition_points_to(PyEdgeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(ControlFlowNode test, ControlFlowNode use | refinement_test(test, use, Conditionals::branchEvaluatesTo(test, use, context, value, origin.toCfgNode()), def) ) @@ -369,7 +405,7 @@ cached module PointsTo2 { /** Holds if ESSA definition, `uniphi`, refers to `(value, origin)`. */ pragma [noinline] - private predicate uni_edged_phi_pointsTo(SingleSuccessorGuard uniphi, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + private predicate uni_edged_phi_points_to(SingleSuccessorGuard uniphi, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(ControlFlowNode test, ControlFlowNode use | /* Because calls such as `len` may create a new variable, we need to go via the source variable * That is perfectly safe as we are only dealing with calls that do not mutate their arguments. @@ -382,19 +418,19 @@ cached module PointsTo2 { /** Points-to for normal assignments `def = ...`. */ pragma [noinline] - private predicate assignment_pointsTo(AssignmentDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate assignment_points_to(AssignmentDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { pointsTo(def.getValue(), context, value, origin) } /** Points-to for deletion: `del name`. */ pragma [noinline] - private predicate delete_pointsTo(DeletionDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate delete_points_to(DeletionDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { value = ObjectInternal::undefined() and origin = def.getDefiningNode() and context.appliesToScope(def.getScope()) } /** Implicit "definition" of `__name__` at the start of a module. */ pragma [noinline] - private predicate module_name_pointsTo(ScopeEntryDefinition def, PointsToContext context, StringObjectInternal value, ControlFlowNode origin) { + private predicate module_name_points_to(ScopeEntryDefinition def, PointsToContext context, StringObjectInternal value, ControlFlowNode origin) { def.getVariable().getName() = "__name__" and exists(Module m | m = def.getScope() @@ -418,26 +454,26 @@ cached module PointsTo2 { /** Holds if the phi-function `phi` refers to `(value, origin)` given the context `context`. */ pragma [nomagic] - private predicate ssa_phi_pointsTo(PhiFunction phi, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + private predicate ssa_phi_points_to(PhiFunction phi, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(EssaVariable input, BasicBlock pred | input = phi.getInput(pred) and - ssa_variable_pointsTo(input, context, value, origin) + variablePointsTo(input, context, value, origin) | controlledReachableEdge(pred, phi.getBasicBlock(), context) or not exists(ConditionBlock guard | guard.controlsEdge(pred, phi.getBasicBlock(), _)) ) or - ssa_variable_pointsTo(phi.getShortCircuitInput(), context, value, origin) + variablePointsTo(phi.getShortCircuitInput(), context, value, origin) } /** Points-to for implicit variable declarations at scope-entry. */ pragma [noinline] - private predicate scope_entry_pointsTo(ScopeEntryDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate scope_entry_points_to(ScopeEntryDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { /* Transfer from another scope */ exists(EssaVariable var, PointsToContext outer, CfgOrigin orig | InterProceduralPointsTo::scope_entry_value_transfer(var, outer, def, context) and - ssa_variable_pointsTo(var, outer, value, orig) and + variablePointsTo(var, outer, value, orig) and origin = orig.asCfgNodeOrHere(def.getDefiningNode()) ) or @@ -462,7 +498,7 @@ cached module PointsTo2 { ) } - private predicate subscript_pointsTo(SubscriptNode sub, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate subscript_points_to(SubscriptNode sub, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { pointsTo(sub.getValue(), context, ObjectInternal::unknown(), _) and value = ObjectInternal::unknown() and origin = sub } @@ -470,21 +506,21 @@ cached module PointsTo2 { /** Track bitwise expressions so we can handle integer flags and enums. * Tracking too many binary expressions is likely to kill performance. */ - private predicate binary_expr_pointsTo(BinaryExprNode b, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate binary_expr_points_to(BinaryExprNode b, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { // TO DO... // Track some integer values through `|` and the types of some objects none() } pragma [noinline] - private predicate compare_expr_pointsTo(CompareNode cmp, PointsToContext context, ObjectInternal value) { + private predicate compare_expr_points_to(CompareNode cmp, PointsToContext context, ObjectInternal value) { value = ObjectInternal::bool(Conditionals::comparisonEvaluatesTo(cmp, _, context, _, _)) // or // value = version_tuple_compare(cmp, context) } pragma [noinline] - private predicate unary_pointsTo(UnaryExprNode f, PointsToContext context, ObjectInternal value) { + private predicate unary_points_to(UnaryExprNode f, PointsToContext context, ObjectInternal value) { exists(Unaryop op, ObjectInternal operand | op = f.getNode().getOp() and pointsTo(f.getOperand(), context, operand, _) @@ -502,7 +538,7 @@ cached module PointsTo2 { module InterModulePointsTo { pragma [noinline] - predicate import_pointsTo(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + predicate import_points_to(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(string name, ImportExpr i | i.getAFlowNode() = f and i.getImportedModuleName() = name and module_imported_as(value, name) and @@ -511,23 +547,23 @@ module InterModulePointsTo { ) } - predicate from_import_pointsTo(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - from_self_import_pointsTo(f, context, value, origin) + predicate from_import_points_to(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + from_self_import_points_to(f, context, value, origin) or - from_other_import_pointsTo(f, context, value, origin) + from_other_import_points_to(f, context, value, origin) } pragma [noinline] - predicate from_self_import_pointsTo(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + predicate from_self_import_points_to(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(EssaVariable var, CfgOrigin orig | var = ssa_variable_for_module_attribute(f, context) and - PointsTo2::ssa_variable_pointsTo(var, context, value, orig) and + PointsToInternal::variablePointsTo(var, context, value, orig) and origin = orig.asCfgNodeOrHere(f) ) } pragma [noinline] - predicate from_other_import_pointsTo(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + predicate from_other_import_points_to(ImportMemberNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(string name, ModuleObjectInternal mod, CfgOrigin orig | from_import_imports(f, context, mod, name) and (mod.getSourceModule() != f.getEnclosingModule() or mod.isBuiltin()) and @@ -538,20 +574,20 @@ module InterModulePointsTo { //not exists(EssaVariable var | var.getSourceVariable().getName() = name and var.getAUse() = f) and //exists(EssaVariable dollar | // isModuleStateVariable(dollar) and dollar.getAUse() = f and - // SSA::ssa_variable_named_attribute_pointsTo(dollar, context, name, value, orig) + // SSA::ssa_variable_named_attribute_points_to(dollar, context, name, value, orig) //) ) } private predicate from_import_imports(ImportMemberNode f, PointsToContext context, ModuleObjectInternal mod, string name) { - PointsTo2::pointsTo(f.getModule(name), context, mod, _) + PointsToInternal::pointsTo(f.getModule(name), context, mod, _) } pragma [noinline] private EssaVariable ssa_variable_for_module_attribute(ImportMemberNode f, PointsToContext context) { exists(string name, ModuleObjectInternal mod, Module m | mod.getSourceModule() = m and m = f.getEnclosingModule() and m = result.getScope() and - PointsTo2::pointsTo(f.getModule(name), context, mod, _) and + PointsToInternal::pointsTo(f.getModule(name), context, mod, _) and result.getSourceVariable().getName() = name and result.getAUse() = f ) } @@ -566,11 +602,11 @@ module InterModulePointsTo { /* Use previous points-to here to avoid slowing down the recursion too much */ exists(SubscriptNode sub | sub.getValue() = sys_modules_flow and - PointsTo2::pointsTo(sys_modules_flow, _, ObjectInternal::sysModules(), _) and + PointsToInternal::pointsTo(sys_modules_flow, _, ObjectInternal::sysModules(), _) and sub.getIndex() = n and n.getNode().(StrConst).getText() = name and sub.(DefinitionNode).getValue() = mod and - PointsTo2::pointsTo(mod, _, m, _) + PointsToInternal::pointsTo(mod, _, m, _) ) ) } @@ -580,7 +616,7 @@ module InterModulePointsTo { * PointsTo isn't exactly how the interpreter works, but is the best approximation we can manage statically. */ pragma [noinline] - predicate implicit_submodule_pointsTo(ImplicitSubModuleDefinition def, PointsToContext context, ModuleObjectInternal value, ControlFlowNode origin) { + predicate implicit_submodule_points_to(ImplicitSubModuleDefinition def, PointsToContext context, ModuleObjectInternal value, ControlFlowNode origin) { exists(PackageObjectInternal package | package.getSourceModule() = def.getDefiningNode().getScope() | value = package.submodule(def.getSourceVariable().getName()) and @@ -590,12 +626,12 @@ module InterModulePointsTo { } /** Points-to for `from ... import *`. */ - predicate import_star_pointsTo(ImportStarRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + predicate import_star_points_to(ImportStarRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(CfgOrigin orig | origin = orig.fix(def.getDefiningNode()) | exists(ModuleObjectInternal mod, string name | - PointsTo2::pointsTo(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) and + PointsToInternal::pointsTo(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) and name = def.getSourceVariable().getName() | /* Attribute from imported module */ module_exports_boolean(mod, name) = true and @@ -605,7 +641,7 @@ module InterModulePointsTo { exists(EssaVariable var | /* Retain value held before import */ variable_not_redefined_by_import_star(var, context, def) and - PointsTo2::ssa_variable_pointsTo(var, context, value,orig) + PointsToInternal::variablePointsTo(var, context, value,orig) ) ) } @@ -615,7 +651,7 @@ module InterModulePointsTo { cached predicate variable_not_redefined_by_import_star(EssaVariable var, PointsToContext context, ImportStarRefinement def) { var = def.getInput() and exists(ModuleObjectInternal mod | - PointsTo2::pointsTo(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) | + PointsToInternal::pointsTo(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) | module_exports_boolean(mod, var.getSourceVariable().getName()) = false or exists(Module m, string name | @@ -627,7 +663,7 @@ module InterModulePointsTo { private predicate importsByImportStar(ModuleObjectInternal mod, ModuleObjectInternal imported) { exists(ImportStarNode isn | - PointsTo2::pointsTo(isn.getModule(), _, imported, _) and + PointsToInternal::pointsTo(isn.getModule(), _, imported, _) and isn.getScope() = mod.getSourceModule() ) or exists(ModuleObjectInternal mid | @@ -660,7 +696,7 @@ module InterModulePointsTo { else ( exists(ImportStarNode isn, ModuleObjectInternal imported | isn.getScope() = src and - PointsTo2::pointsTo(isn.getModule(), _, imported, _) and + PointsToInternal::pointsTo(isn.getModule(), _, imported, _) and result = module_exports_boolean(imported, name) ) or @@ -686,9 +722,9 @@ module InterModulePointsTo { module InterProceduralPointsTo { pragma [noinline] - predicate call_pointsTo(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + predicate call_points_to(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(ObjectInternal func, CfgOrigin resultOrigin | - PointsTo2::pointsTo(f.getFunction(), context, func, _) and + PointsToInternal::pointsTo(f.getFunction(), context, func, _) and origin = resultOrigin.fix(f) | exists(PointsToContext callee | @@ -702,21 +738,21 @@ module InterProceduralPointsTo { /** Points-to for parameter. `def foo(param): ...`. */ pragma [noinline] - predicate parameter_pointsTo(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - positional_parameter_pointsTo(def, context, value, origin) + predicate parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + positional_parameter_points_to(def, context, value, origin) or - named_parameter_pointsTo(def, context, value, origin) + named_parameter_points_to(def, context, value, origin) or - default_parameter_pointsTo(def, context, value, origin) + default_parameter_points_to(def, context, value, origin) or - special_parameter_pointsTo(def, context, value, origin) + special_parameter_points_to(def, context, value, origin) } - /** Helper for `parameter_pointsTo` */ + /** Helper for `parameter_points_to` */ pragma [noinline] - private predicate positional_parameter_pointsTo(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate positional_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(PointsToContext caller, ControlFlowNode arg | - PointsTo2::pointsTo(arg, caller, value, origin) and + PointsToInternal::pointsTo(arg, caller, value, origin) and callsite_argument_transfer(arg, caller, def, context) ) or @@ -725,23 +761,23 @@ module InterProceduralPointsTo { } - /** Helper for `parameter_pointsTo` */ + /** Helper for `parameter_points_to` */ pragma [noinline] - private predicate named_parameter_pointsTo(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate named_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(CallNode call, PointsToContext caller, PythonFunctionObjectInternal func, string name | context.fromCall(call, func, caller) and def.getParameter() = func.getScope().getArgByName(name) and - PointsTo2::pointsTo(call.getArgByName(name), caller, value, origin) + PointsToInternal::pointsTo(call.getArgByName(name), caller, value, origin) ) } - /** Helper for parameter_pointsTo */ - private predicate default_parameter_pointsTo(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - exists(PointsToContext imp | imp.isImport() | PointsTo2::pointsTo(def.getDefault(), imp, value, origin)) and + /** Helper for parameter_points_to */ + private predicate default_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + exists(PointsToContext imp | imp.isImport() | PointsToInternal::pointsTo(def.getDefault(), imp, value, origin)) and context_for_default_value(def, context) } - /** Helper for default_parameter_pointsTo */ + /** Helper for default_parameter_points_to */ pragma [noinline] private predicate context_for_default_value(ParameterDefinition def, PointsToContext context) { context.isRuntime() @@ -756,9 +792,9 @@ module InterProceduralPointsTo { ) } - /** Helper for parameter_pointsTo */ + /** Helper for parameter_points_to */ pragma [noinline] - private predicate special_parameter_pointsTo(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate special_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { context.isRuntime() and origin = def.getDefiningNode() and exists(ControlFlowNode param | @@ -792,7 +828,7 @@ module InterProceduralPointsTo { cached predicate callsite_calls_function(CallNode call, PointsToContext caller, Function scope, PointsToContext callee, int parameter_offset) { callee.fromCall(call, caller) and exists(ObjectInternal func | - PointsTo2::pointsTo(call.getFunction(), caller, func, _) and + PointsToInternal::pointsTo(call.getFunction(), caller, func, _) and func.calleeAndOffset(scope, parameter_offset) ) } @@ -861,7 +897,7 @@ module InterProceduralPointsTo { * Where var may be redefined in call to `foo` if `var` escapes (is global or non-local). */ pragma [noinline] - predicate callsite_pointsTo(CallsiteRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + predicate callsite_points_to(CallsiteRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { exists(SsaSourceVariable srcvar | srcvar = def.getSourceVariable() | if srcvar instanceof EscapingAssignmentGlobalVariable then ( @@ -869,17 +905,17 @@ module InterProceduralPointsTo { exists(EssaVariable var, Function func, PointsToContext callee | callsite_calls_function(def.getCall(), context, func, callee, _) and var_at_exit(srcvar, func, var) and - PointsTo2::ssa_variable_pointsTo(var, callee, value, origin) + PointsToInternal::variablePointsTo(var, callee, value, origin) ) or exists(ObjectInternal callable | - PointsTo2::pointsTo(def.getCall().getFunction(), context, callable, _) and + PointsToInternal::pointsTo(def.getCall().getFunction(), context, callable, _) and exists(callable.getBuiltin()) and - PointsTo2::ssa_variable_pointsTo(def.getInput(), context, value, origin) + PointsToInternal::variablePointsTo(def.getInput(), context, value, origin) ) ) else ( /* Otherwise we can assume its value (but not those of its attributes or members) has not changed. */ - PointsTo2::ssa_variable_pointsTo(def.getInput(), context, value, origin) + PointsToInternal::variablePointsTo(def.getInput(), context, value, origin) ) ) } @@ -896,7 +932,7 @@ module InterProceduralPointsTo { /** Gets the `value, origin` that `f` would refer to if it has not been assigned some other value */ pragma [noinline] -private predicate potential_builtin_pointsTo(NameNode f, ObjectInternal value, ControlFlowNode origin) { +private predicate potential_builtin_points_to(NameNode f, ObjectInternal value, ControlFlowNode origin) { f.isGlobal() and f.isLoad() and origin = f and ( value = ObjectInternal::builtin(f.getId()) @@ -915,7 +951,7 @@ private module Conditionals { boolean branchEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { contains_interesting_expression_within_test(expr, use) and - PointsTo2::pointsTo(use, context, val, origin) and + PointsToInternal::pointsTo(use, context, val, origin) and expr = use and val.booleanValue() = result or @@ -943,8 +979,8 @@ private module Conditionals { exists(ControlFlowNode r, boolean sense | equality_test(expr, use, sense, r) and exists(ObjectInternal other | - PointsTo2::pointsTo(use, context, val, origin) and - PointsTo2::pointsTo(r, context, other, _) | + PointsToInternal::pointsTo(use, context, val, origin) and + PointsToInternal::pointsTo(r, context, other, _) | val.isComparable() = true and other.isComparable() = true and ( other = val and result = sense @@ -968,8 +1004,8 @@ private module Conditionals { or inequality(expr, r, use, strict) and sense = false ) and - PointsTo2::pointsTo(use, context, val, origin) and - PointsTo2::pointsTo(r, context, other, _) + PointsToInternal::pointsTo(use, context, val, origin) and + PointsToInternal::pointsTo(r, context, other, _) | val.intValue() < other.intValue() and result = sense or @@ -1028,7 +1064,7 @@ cached module Types { or exists(Class pycls | pycls = cls.(PythonClassObjectInternal).getScope() | - PointsTo2::pointsTo(pycls.getBase(n).getAFlowNode(), _, result, _) + PointsToInternal::pointsTo(pycls.getBase(n).getAFlowNode(), _, result, _) or not exists(pycls.getABase()) and n = 0 and isNewStyle(cls) and result = ObjectInternal::builtin("object") @@ -1063,7 +1099,7 @@ cached module Types { exists(EssaVariable var | name = var.getName() and var.getAUse() = cls.(PythonClassObjectInternal).getScope().getANormalExit() and - PointsTo2::ssa_variable_pointsTo(var, _, value, origin) + PointsToInternal::variablePointsTo(var, _, value, origin) ) } @@ -1075,7 +1111,7 @@ cached module Types { private ClassObjectInternal declaredMetaClass(PythonClassObjectInternal cls) { exists(ObjectInternal obj | - PointsTo2::ssa_variable_pointsTo(metaclass_var(cls.getScope()), _, obj, _) | + PointsToInternal::variablePointsTo(metaclass_var(cls.getScope()), _, obj, _) | result = obj or obj = ObjectInternal::unknown() and result = ObjectInternal::unknownClass() @@ -1083,7 +1119,7 @@ cached module Types { or exists(ControlFlowNode meta | six_add_metaclass(_, cls, meta) and - PointsTo2::pointsTo(meta, _, result, _) + PointsToInternal::pointsTo(meta, _, result, _) ) } @@ -1098,7 +1134,7 @@ cached module Types { private boolean has_metaclass_var_metaclass(PythonClassObjectInternal cls) { exists(ObjectInternal obj | - PointsTo2::ssa_variable_pointsTo(metaclass_var(cls.getScope()), _, obj, _) | + PointsToInternal::variablePointsTo(metaclass_var(cls.getScope()), _, obj, _) | obj = ObjectInternal::undefined() and result = false or obj != ObjectInternal::undefined() and result = true @@ -1126,11 +1162,11 @@ cached module Types { // decorator_call.getArg(0) = decorated and // decorator = decorator_call.getFunction() and // decorator.getArg(0) = metaclass | - // PointsTo2::pointsTo(decorator.getFunction(), _, six_add_metaclass_function(), _) + // PointsToInternal::pointsTo(decorator.getFunction(), _, six_add_metaclass_function(), _) // or // exists(ModuleObjectInternal six | // six.getName() = "six" and - // PointsTo2::pointsTo(decorator.getFunction().(AttrNode).getObject("add_metaclass"), _, six, _) + // PointsToInternal::pointsTo(decorator.getFunction().(AttrNode).getObject("add_metaclass"), _, six, _) // ) //) } diff --git a/python/ql/src/semmle/python/pointsto/PointsToContext.qll b/python/ql/src/semmle/python/pointsto/PointsToContext.qll index 592e3c06b348..2f9f34c81fb1 100644 --- a/python/ql/src/semmle/python/pointsto/PointsToContext.qll +++ b/python/ql/src/semmle/python/pointsto/PointsToContext.qll @@ -1,5 +1,5 @@ import python -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.objects.ObjectInternal /* * A note on 'cost'. Cost doesn't represent the cost to compute, @@ -149,7 +149,7 @@ class PointsToContext extends TPointsToContext { /** Holds if `call` is the call-site from which this context was entered and `caller` is the caller's context. */ predicate fromCall(CallNode call, PythonFunctionObjectInternal callee, PointsToContext caller) { - call = PointsTo2::get_a_call(callee, caller) and + call = callee.getACall(caller) and this = TCallContext(call, caller, _) } @@ -170,7 +170,7 @@ class PointsToContext extends TPointsToContext { or /* Called functions, regardless of their name */ exists(CallableObjectInternal callable, ControlFlowNode call, TPointsToContext outerContext | - call = PointsTo2::get_a_call(callable, outerContext) and + call = callable.getACall(outerContext) and this = TCallContext(call, outerContext, _) | s = callable.getScope() ) diff --git a/python/ql/src/semmle/python/types/ClassObject.qll b/python/ql/src/semmle/python/types/ClassObject.qll index faa2da56f611..ed34eff58142 100644 --- a/python/ql/src/semmle/python/types/ClassObject.qll +++ b/python/ql/src/semmle/python/types/ClassObject.qll @@ -1,7 +1,7 @@ import python private import semmle.python.objects.Classes private import semmle.python.objects.Instances -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.MRO2 private import semmle.python.types.Builtins diff --git a/python/ql/src/semmle/python/types/Descriptors.qll b/python/ql/src/semmle/python/types/Descriptors.qll index 2b183c5445ac..74f443624dd9 100644 --- a/python/ql/src/semmle/python/types/Descriptors.qll +++ b/python/ql/src/semmle/python/types/Descriptors.qll @@ -1,5 +1,4 @@ import python -private import semmle.python.pointsto.PointsTo2 private import semmle.python.objects.ObjectInternal diff --git a/python/ql/src/semmle/python/types/FunctionObject.qll b/python/ql/src/semmle/python/types/FunctionObject.qll index 09889cfd4230..4f210b1548d6 100644 --- a/python/ql/src/semmle/python/types/FunctionObject.qll +++ b/python/ql/src/semmle/python/types/FunctionObject.qll @@ -1,6 +1,6 @@ import python import semmle.python.types.Exceptions -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.objects.Callables private import semmle.python.libraries.Zope private import semmle.python.pointsto.Base @@ -64,7 +64,7 @@ abstract class FunctionObject extends Object { /** Gets a call-site from where this function is called, given the `context` */ ControlFlowNode getACall(Context caller_context) { - result = PointsTo2::get_a_call(theCallable(), caller_context) + result = theCallable().getACall(caller_context) } /** Gets the `ControlFlowNode` that will be passed as the nth argument to `this` when called at `call`. diff --git a/python/ql/src/semmle/python/types/ModuleObject.qll b/python/ql/src/semmle/python/types/ModuleObject.qll index 90076fce7d97..78376c35d4e6 100644 --- a/python/ql/src/semmle/python/types/ModuleObject.qll +++ b/python/ql/src/semmle/python/types/ModuleObject.qll @@ -1,5 +1,5 @@ import python -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo private import semmle.python.objects.Modules private import semmle.python.types.ModuleKind From 610a35c1871147005f1e448bb0ddd8ad5f5d4f03 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 28 Mar 2019 09:26:33 +0000 Subject: [PATCH 022/128] Python points-to: Improve backwards compatibility for comparisons. --- .../ql/src/semmle/python/objects/Classes.qll | 5 +- .../semmle/python/objects/ObjectInternal.qll | 4 +- .../src/semmle/python/pointsto/PointsTo.qll | 158 +++++++++++++++--- .../src/semmle/python/types/Descriptors.qll | 4 + 4 files changed, 149 insertions(+), 22 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 297208171f6b..5dc012169f79 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -129,6 +129,9 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec override ObjectInternal getClass() { result = TBuiltinClassObject(this.getBuiltin().getClass()) + or + this.getBuiltin().getClass() = Builtin::special("type") and + result = TType() } override ControlFlowNode getOrigin() { @@ -163,7 +166,7 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { override string toString() { - none() + result = "Unknown class" } override ClassDecl getClassDeclaration() { diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 38f36fcb4368..64726c89719c 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -174,7 +174,7 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { class UnknownInternal extends ObjectInternal, TUnknown { override string toString() { - none() + result = "Unknown value" } override boolean booleanValue() { @@ -242,7 +242,7 @@ class UnknownInternal extends ObjectInternal, TUnknown { class UndefinedInternal extends ObjectInternal, TUndefined { override string toString() { - none() + result = "Undefined variable" } override boolean booleanValue() { diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index b3a1935e4323..6af110563df9 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -109,8 +109,14 @@ module PointsTo { points_to(ControlFlowNode f, PointsToContext context, Object obj, ClassObject cls, ControlFlowNode origin) { exists(Value value | PointsToInternal::pointsTo(f, context, value, origin) and - obj = value.getSource() and - cls = value.getClass().getSource() + cls = value.getClass().getSource() | + obj = value.getSource() or + not exists(value.getSource()) and obj = origin + ) + or + f.isParameter() and exists(EssaVariable var | + var.getDefinition().(ParameterDefinition).getDefiningNode() = f and + ssa_variable_points_to(var, context, obj, cls, origin) ) } @@ -118,8 +124,9 @@ module PointsTo { ssa_variable_points_to(EssaVariable var, PointsToContext context, Object obj, ClassObject cls, CfgOrigin origin) { exists(Value value | PointsToInternal::variablePointsTo(var, context, value, origin) and - obj = value.getSource() and - cls = value.getClass().getSource() + cls = value.getClass().getSource() | + obj = value.getSource() or + not exists(value.getSource()) and obj = origin ) } @@ -153,10 +160,12 @@ cached module PointsToInternal { or binary_expr_points_to(f, context, value, origin) or - origin = f and compare_expr_points_to(f, context, value) + origin = f and test_expr_points_to(f, context, value) or origin = f and unary_points_to(f, context, value) or + if_exp_points_to(f, context, value, origin) + or origin = f and value.introduced(f, context) or InterModulePointsTo::import_points_to(f, context, value, origin) @@ -375,8 +384,8 @@ cached module PointsToInternal { //or InterModulePointsTo::import_star_points_to(def, context, value, origin) or - //attribute_assignment_points_to(def, context, value, origin) - //or + attribute_assignment_points_to(def, context, value, origin) + or InterProceduralPointsTo::callsite_points_to(def, context, value, origin) or argument_points_to(def, context, value, origin) @@ -386,6 +395,18 @@ cached module PointsToInternal { uni_edged_phi_points_to(def, context, value, origin) } + /** Attribute assignments have no effect as far as value tracking is concerned, except for `__class__`. */ + pragma [noinline] + private predicate attribute_assignment_points_to(AttributeAssignment def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + if def.getName() = "__class__" then + exists(ObjectInternal cls | + pointsTo(def.getValue(), context, cls, _) and + value = TUnknownInstance(cls) + ) + else + variablePointsTo(def.getInput(), context, value, origin) + } + /** Ignore the effects of calls on their arguments. PointsTo is an approximation, but attempting to improve accuracy would be very expensive for very little gain. */ private predicate argument_points_to(ArgumentRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { variablePointsTo(def.getInput(), context, value, origin) @@ -513,8 +534,8 @@ cached module PointsToInternal { } pragma [noinline] - private predicate compare_expr_points_to(CompareNode cmp, PointsToContext context, ObjectInternal value) { - value = ObjectInternal::bool(Conditionals::comparisonEvaluatesTo(cmp, _, context, _, _)) + private predicate test_expr_points_to(ControlFlowNode cmp, PointsToContext context, ObjectInternal value) { + value = ObjectInternal::bool(Conditionals::testEvaluatesTo(cmp, _, context, _, _)) // or // value = version_tuple_compare(cmp, context) } @@ -533,6 +554,11 @@ cached module PointsToInternal { ) } + /** Holds if `f` is an expression node `tval if cond else fval` and points to `(value, origin)`. */ + private predicate if_exp_points_to(IfExprNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + pointsTo(f.getAnOperand(), context, value, origin) + } + } module InterModulePointsTo { @@ -941,7 +967,7 @@ private predicate potential_builtin_points_to(NameNode f, ObjectInternal value, ) } -private module Conditionals { +module Conditionals { /** Holds if `expr` is the operand of a unary `not` expression. */ private ControlFlowNode not_operand(ControlFlowNode expr) { @@ -952,22 +978,33 @@ private module Conditionals { boolean branchEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { contains_interesting_expression_within_test(expr, use) and PointsToInternal::pointsTo(use, context, val, origin) and - expr = use and - val.booleanValue() = result + ( + expr = use and + val.booleanValue() = result + or + exists(string name, ObjectInternal attr | + expr.(AttrNode).getObject(name) = use | + val.attribute(name, attr, _) and + result = attr.booleanValue() + or + val.attributesUnknown() and + result = maybe() + ) + ) or - result = comparisonEvaluatesTo(expr, use, context, val, origin) + result = testEvaluatesTo(expr, use, context, val, origin) or result = branchEvaluatesTo(not_operand(expr), use, context, val, origin).booleanNot() } - boolean comparisonEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { + boolean testEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { result = equalityEvaluatesTo(expr, use, context, val, origin) or result = inequalityEvaluatesTo(expr, use, context, val, origin) - // or - //result = isinstance_test_evaluates_boolean(expr, use, context, val, origin) - //or - //result = issubclass_test_evaluates_boolean(expr, use, context, val, origin) + or + result = isinstance_test_evaluates_boolean(expr, use, context, val, origin) + or + result = issubclass_test_evaluates_boolean(expr, use, context, val, origin) //or //result = callable_test_evaluates_boolean(expr, use, context, val, origin) //or @@ -995,6 +1032,64 @@ private module Conditionals { ) } + private predicate isinstance_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls, ControlFlowNode origin) { + PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("isinstance"), _) and + use = call.getArg(0) and + PointsToInternal::pointsTo(use, context, val, origin) and + PointsToInternal::pointsTo(call.getArg(1), context, cls, _) + } + + pragma [nomagic] + private boolean isinstance_test_evaluates_boolean(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { + exists(ObjectInternal cls | + isinstance_call(call, use, context, val, cls, origin) | + result = Types::improperSubclass(val.getClass(), cls) + or + val = ObjectInternal::unknown() and result = maybe() + or + cls = ObjectInternal::unknown() and result = maybe() + or + cls = ObjectInternal::unknownClass() and result = maybe() + ) + } + + private predicate issubclass_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls, ControlFlowNode origin) { + PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("issubclass"), _) and + use = call.getArg(0) and + PointsToInternal::pointsTo(use, context, val, origin) and + PointsToInternal::pointsTo(call.getArg(1), context, cls, _) + } + + pragma [nomagic] + private boolean issubclass_test_evaluates_boolean(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { + exists(ObjectInternal cls | + issubclass_call(call, use, context, val, cls, origin) | + result = Types::improperSubclass(val, cls) + or + val = ObjectInternal::unknownClass() and result = maybe() + or + val = ObjectInternal::unknown() and result = maybe() + or + cls = ObjectInternal::unknown() and result = maybe() + or + cls = ObjectInternal::unknownClass() and result = maybe() + ) + } + + predicate requireSubClass(ObjectInternal sub, ObjectInternal sup) { + sup != ObjectInternal::unknownClass() and + sub != ObjectInternal::unknownClass() and + (sup.isClass() = true or sup instanceof TupleObjectInternal) and + ( + issubclass_call(_, _, _, sub, sup, _) and sub.isClass() = true + or + exists(ObjectInternal val | + isinstance_call(_, _, _, val, sup, _) and + sub = val.getClass() + ) + ) + } + pragma [noinline] private boolean inequalityEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { exists(ControlFlowNode r, boolean sense | @@ -1248,7 +1343,32 @@ cached module Types { exists(int i, int j | getBase(cls, i) = getBase(cls, j) and i != j and reason = "Duplicate bases classes") } -} + cached boolean improperSubclass(ObjectInternal sub, ObjectInternal sup) { + sub = sup and result = true + or + result = mroContains(Types::getMro(sub), sup) + // TO DO... + // Handle tuples of classes + } + + private boolean mroContains(ClassList mro, ObjectInternal sup) { + result = mroContains(mro, sup, 0) + } + private boolean mroContains(ClassList mro, ObjectInternal sup, int n) { + exists(ClassObjectInternal cls | + Conditionals::requireSubClass(cls, sup) and + mro = getMro(cls) + ) + and + ( + n = mro.length() and result = false + or + mro.getItem(n) = sup and result = true + or + mro.getItem(n) != sup and result = mroContains(mro, sup, n+1) + ) + } +} diff --git a/python/ql/src/semmle/python/types/Descriptors.qll b/python/ql/src/semmle/python/types/Descriptors.qll index 74f443624dd9..949ff1d42660 100644 --- a/python/ql/src/semmle/python/types/Descriptors.qll +++ b/python/ql/src/semmle/python/types/Descriptors.qll @@ -16,6 +16,10 @@ class ClassMethodObject extends Object { ) } + CallNode getACall() { + result = this.getFunction().getACall() + } + } /** A static method object. Either a decorated function or an explicit call to staticmethod(f) */ From 8af6cb66441d97f204cf8d964efb34dcc17f264d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 28 Mar 2019 13:24:00 +0000 Subject: [PATCH 023/128] Python points-to: Use objects, not booleans when doing evaluation of tests. --- .../src/semmle/python/objects/Callables.qll | 22 ++- .../ql/src/semmle/python/objects/Classes.qll | 4 +- python/ql/src/semmle/python/pointsto/Base.qll | 4 + .../ql/src/semmle/python/pointsto/Filters.qll | 14 -- .../src/semmle/python/pointsto/PointsTo.qll | 177 ++++++++++++------ 5 files changed, 140 insertions(+), 81 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 7ac02a635048..259496f702be 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -87,13 +87,9 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti rval = func.getAReturnValueFlowNode() and PointsToInternal::pointsTo(rval, callee, obj, origin) or - exists(Return ret | - ret.getScope() = func and - PointsToInternal::reachableBlock(ret.getAFlowNode().getBasicBlock(), callee) and - not exists(ret.getValue()) and - obj = ObjectInternal::none_() and - origin = CfgOrigin::unknown() - ) + PointsToInternal::reachableBlock(blockReturningNone(func), callee) and + obj = ObjectInternal::none_() and + origin = CfgOrigin::unknown() ) } @@ -121,12 +117,22 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti PointsTo::pointsTo(result.getFunction(), ctx, this, _) or exists(BoundMethodObjectInternal bm | - bm.getACall() = result and this = bm.getFunction() + bm.getACall(ctx) = result and this = bm.getFunction() ) } } + +private BasicBlock blockReturningNone(Function func) { + exists(Return ret | + not exists(ret.getValue()) and + ret.getScope() = func and + result = ret.getAFlowNode().getBasicBlock() + ) +} + + class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunctionObject { override Builtin getBuiltin() { diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 5dc012169f79..896718fd6e64 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -214,7 +214,7 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { class TypeInternal extends ClassObjectInternal, TType { override string toString() { - result = "type" + result = "builtin-class type" } override ClassDecl getClassDeclaration() { @@ -229,7 +229,7 @@ class TypeInternal extends ClassObjectInternal, TType { none() } - override boolean isComparable() { result = false } + override boolean isComparable() { result = true } override Builtin getBuiltin() { result = Builtin::special("type") diff --git a/python/ql/src/semmle/python/pointsto/Base.qll b/python/ql/src/semmle/python/pointsto/Base.qll index ed17ad4b4292..7bbba90979df 100644 --- a/python/ql/src/semmle/python/pointsto/Base.qll +++ b/python/ql/src/semmle/python/pointsto/Base.qll @@ -395,6 +395,10 @@ class SingleSuccessorGuard extends PyNodeRefinement { not exists(this.getSense()) and result = PyNodeRefinement.super.getRepresentation() + " [??]" } + + ControlFlowNode getTest() { + result = this.getDefiningNode() + } } /** Implicit definition of the names of sub-modules in a package. diff --git a/python/ql/src/semmle/python/pointsto/Filters.qll b/python/ql/src/semmle/python/pointsto/Filters.qll index 6ac515b217a4..c69b72b70965 100644 --- a/python/ql/src/semmle/python/pointsto/Filters.qll +++ b/python/ql/src/semmle/python/pointsto/Filters.qll @@ -25,20 +25,6 @@ predicate isinstance(CallNode fc, ControlFlowNode cls, ControlFlowNode use) { cls = fc.getArg(1) and fc.getArg(0) = use } -/** Holds if `c` is a test comparing `x` and `y`. `is` is true if the operator is `is` or `==`, it is false if the operator is `is not` or `!=`. */ -predicate equality_test(CompareNode c, ControlFlowNode x, boolean is, ControlFlowNode y) { - exists(Cmpop op | - c.operands(x, op, y) or - c.operands(y, op, x) - | - (is = true and op instanceof Is or - is = false and op instanceof IsNot or - is = true and op instanceof Eq or - is = false and op instanceof NotEq - ) - ) -} - /** Holds if `c` is a call to `issubclass(use, cls)`. */ predicate issubclass(CallNode fc, ControlFlowNode cls, ControlFlowNode use) { fc.getFunction().(NameNode).getId() = "issubclass" and diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 6af110563df9..6c6aae8d2116 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -319,10 +319,13 @@ cached module PointsToInternal { pragma [noinline] private predicate attribute_load_points_to(AttrNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { f.isLoad() and - exists(ObjectInternal object, string name, CfgOrigin orig | - pointsTo(f.getObject(name), context, object, _) | - object.attribute(name, value, orig) and - origin = orig.fix(f) + exists(ObjectInternal object, string name | + pointsTo(f.getObject(name), context, object, _) + | + exists(CfgOrigin orig | + object.attribute(name, value, orig) and + origin = orig.fix(f) + ) or object.attributesUnknown() and origin = f and value = ObjectInternal::unknown() @@ -401,7 +404,8 @@ cached module PointsToInternal { if def.getName() = "__class__" then exists(ObjectInternal cls | pointsTo(def.getValue(), context, cls, _) and - value = TUnknownInstance(cls) + value = TUnknownInstance(cls) and + origin = CfgOrigin::fromCfgNode(def.getDefiningNode()) ) else variablePointsTo(def.getInput(), context, value, origin) @@ -419,9 +423,10 @@ cached module PointsToInternal { /** Holds if ESSA edge refinement, `def`, refers to `(value, cls, origin)`. */ private predicate ssa_filter_definition_points_to(PyEdgeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - exists(ControlFlowNode test, ControlFlowNode use | - refinement_test(test, use, Conditionals::branchEvaluatesTo(test, use, context, value, origin.toCfgNode()), def) - ) + def.getSense() = Conditionals::testEvaluates(def.getTest(), def.getInput().getASourceUse(), context, value, origin) + //exists(ControlFlowNode test, ControlFlowNode use | + // refinement_test(test, use, Conditionals::branchEvaluatesTo(test, use, context, value, origin.toCfgNode()), def) + //) } /** Holds if ESSA definition, `uniphi`, refers to `(value, origin)`. */ @@ -431,9 +436,9 @@ cached module PointsToInternal { /* Because calls such as `len` may create a new variable, we need to go via the source variable * That is perfectly safe as we are only dealing with calls that do not mutate their arguments. */ - use = uniphi.getInput().getSourceVariable().(Variable).getAUse() and - test = uniphi.getDefiningNode() and - uniphi.getSense() = Conditionals::branchEvaluatesTo(test, use, context, value, origin.toCfgNode()) + use = uniphi.getInput().getASourceUse() and + test = uniphi.getTest() and + uniphi.getSense() = Conditionals::testEvaluates(test, use, context, value, origin.toCfgNode()) ) } @@ -535,7 +540,10 @@ cached module PointsToInternal { pragma [noinline] private predicate test_expr_points_to(ControlFlowNode cmp, PointsToContext context, ObjectInternal value) { - value = ObjectInternal::bool(Conditionals::testEvaluatesTo(cmp, _, context, _, _)) + exists(ControlFlowNode use | + value = Conditionals::evaluates(cmp, use, context, _, _) and + use != cmp + ) // or // value = version_tuple_compare(cmp, context) } @@ -969,70 +977,99 @@ private predicate potential_builtin_points_to(NameNode f, ObjectInternal value, module Conditionals { - /** Holds if `expr` is the operand of a unary `not` expression. */ - private ControlFlowNode not_operand(ControlFlowNode expr) { - expr.(UnaryExprNode).getNode().getOp() instanceof Not and - result = expr.(UnaryExprNode).getOperand() + boolean testEvaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + //pinode_test(expr, use) and + result = evaluates(expr, use, context, value, origin).booleanValue() } - boolean branchEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { - contains_interesting_expression_within_test(expr, use) and + ObjectInternal evaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { PointsToInternal::pointsTo(use, context, val, origin) and - ( - expr = use and - val.booleanValue() = result + pinode_test(_, use) and expr = use and result = val + or + exists(ControlFlowNode part, ObjectInternal partval | + pinode_test_part(expr, part) and + partval = evaluates(part, use, context, val, origin) + | + result = equalityEvaluates(expr, part, context, partval) or - exists(string name, ObjectInternal attr | - expr.(AttrNode).getObject(name) = use | - val.attribute(name, attr, _) and - result = attr.booleanValue() + result = ObjectInternal::bool(inequalityEvaluatesBoolean(expr, part, context, partval)) + or + result = ObjectInternal::bool(isinstance_test_evaluates_boolean(expr, part, context, partval)) + or + result = ObjectInternal::bool(issubclass_test_evaluates_boolean(expr, part, context, partval)) + or + exists(string attr | + expr.(AttrNode).getObject(attr) = use | + val.attribute(attr, result, _) or - val.attributesUnknown() and - result = maybe() + val.attributesUnknown() and result = ObjectInternal::unknown() ) + or + expr instanceof BinaryExprNode and result = ObjectInternal::unknown() + or + part = not_operand(expr) and result = ObjectInternal::bool(partval.booleanValue().booleanNot()) + or + result = evaluatesLen(expr, part, context, partval) + //or + //result = callable_test_evaluates_boolean(expr, use, context, val, origin) + //or + //result = hasattr_test_evaluates_boolean(expr, use, context, val, origin) ) - or - result = testEvaluatesTo(expr, use, context, val, origin) - or - result = branchEvaluatesTo(not_operand(expr), use, context, val, origin).booleanNot() + } - boolean testEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { - result = equalityEvaluatesTo(expr, use, context, val, origin) - or - result = inequalityEvaluatesTo(expr, use, context, val, origin) - or - result = isinstance_test_evaluates_boolean(expr, use, context, val, origin) - or - result = issubclass_test_evaluates_boolean(expr, use, context, val, origin) - //or - //result = callable_test_evaluates_boolean(expr, use, context, val, origin) - //or - //result = hasattr_test_evaluates_boolean(expr, use, context, val, origin) + /** Holds if `expr` is the operand of a unary `not` expression. */ + private ControlFlowNode not_operand(ControlFlowNode expr) { + expr.(UnaryExprNode).getNode().getOp() instanceof Not and + result = expr.(UnaryExprNode).getOperand() } pragma [noinline] - private boolean equalityEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { + private ObjectInternal equalityEvaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val) { exists(ControlFlowNode r, boolean sense | - equality_test(expr, use, sense, r) and + pinode_test_part(expr, use) and equality_test(expr, use, sense, r) and exists(ObjectInternal other | - PointsToInternal::pointsTo(use, context, val, origin) and + PointsToInternal::pointsTo(use, context, val, _) and PointsToInternal::pointsTo(r, context, other, _) | val.isComparable() = true and other.isComparable() = true and ( - other = val and result = sense + other = val and result = ObjectInternal::bool(sense) or - other != val and result = sense.booleanNot() + other != val and result = ObjectInternal::bool(sense.booleanNot()) ) or - val.isComparable() = false and result = maybe() + val.isComparable() = false and result = ObjectInternal::bool(_) or - other.isComparable() = false and result = maybe() + other.isComparable() = false and result = ObjectInternal::bool(_) ) ) } - private predicate isinstance_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls, ControlFlowNode origin) { + /** Holds if `c` is a test comparing `x` and `y`. `is` is true if the operator is `is` or `==`, it is false if the operator is `is not` or `!=`. */ + private predicate equality_test(CompareNode c, ControlFlowNode x, boolean is, ControlFlowNode y) { + exists(Cmpop op | + c.operands(x, op, y) or + c.operands(y, op, x) + | + (is = true and op instanceof Is or + is = false and op instanceof IsNot or + is = true and op instanceof Eq or + is = false and op instanceof NotEq + ) + ) + } + + pragma [noinline] + private ObjectInternal evaluatesLen(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { + pinode_test_part(call, use) and + PointsToInternal::pointsTo(use, context, val, _) and + PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("len"), _) and + result = TInt(val.(SequenceObjectInternal).length()) + } + + //private + predicate isinstance_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls, ControlFlowNode origin) { + pinode_test_part(call, use) and PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("isinstance"), _) and use = call.getArg(0) and PointsToInternal::pointsTo(use, context, val, origin) and @@ -1040,9 +1077,10 @@ module Conditionals { } pragma [nomagic] - private boolean isinstance_test_evaluates_boolean(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { + //private + boolean isinstance_test_evaluates_boolean(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { exists(ObjectInternal cls | - isinstance_call(call, use, context, val, cls, origin) | + isinstance_call(call, use, context, val, cls, _) | result = Types::improperSubclass(val.getClass(), cls) or val = ObjectInternal::unknown() and result = maybe() @@ -1061,9 +1099,10 @@ module Conditionals { } pragma [nomagic] - private boolean issubclass_test_evaluates_boolean(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { + private boolean issubclass_test_evaluates_boolean(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { + pinode_test_part(call, use) and exists(ObjectInternal cls | - issubclass_call(call, use, context, val, cls, origin) | + issubclass_call(call, use, context, val, cls, _) | result = Types::improperSubclass(val, cls) or val = ObjectInternal::unknownClass() and result = maybe() @@ -1091,7 +1130,8 @@ module Conditionals { } pragma [noinline] - private boolean inequalityEvaluatesTo(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { + private boolean inequalityEvaluatesBoolean(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val) { + pinode_test_part(expr, use) and exists(ControlFlowNode r, boolean sense | exists(boolean strict, ObjectInternal other | ( @@ -1099,7 +1139,7 @@ module Conditionals { or inequality(expr, r, use, strict) and sense = false ) and - PointsToInternal::pointsTo(use, context, val, origin) and + PointsToInternal::pointsTo(use, context, val, _) and PointsToInternal::pointsTo(r, context, other, _) | val.intValue() < other.intValue() and result = sense @@ -1135,6 +1175,29 @@ module Conditionals { ) } + private predicate pinode_test(ControlFlowNode test, NameNode use) { + exists(PyEdgeRefinement pi | + pi.getInput().getASourceUse() = use and + pi.getTest() = test and + test.getAChild*() = use + ) + or + exists(SingleSuccessorGuard unipi | + unipi.getInput().getASourceUse() = use and + unipi.getTest() = test and + test.getAChild*() = use + ) + } + + private predicate pinode_test_part(ControlFlowNode outer, ControlFlowNode inner) { + exists(ControlFlowNode test, NameNode use | + pinode_test(test, use) and + test.getAChild*() = outer and + outer.getAChild+() = inner and + inner.getAChild*() = use + ) + } + } cached module Types { From 23ca403728da30ec38411159a3fa021b88b5634f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 28 Mar 2019 14:18:56 +0000 Subject: [PATCH 024/128] Python points-to: Understand callable and hasattr. --- .../src/semmle/python/pointsto/PointsTo.qll | 85 +++++++++++++++++-- 1 file changed, 80 insertions(+), 5 deletions(-) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 6c6aae8d2116..35efc0edd7f0 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -978,7 +978,7 @@ private predicate potential_builtin_points_to(NameNode f, ObjectInternal value, module Conditionals { boolean testEvaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - //pinode_test(expr, use) and + pinode_test(expr, use) and result = evaluates(expr, use, context, value, origin).booleanValue() } @@ -1010,10 +1010,10 @@ module Conditionals { part = not_operand(expr) and result = ObjectInternal::bool(partval.booleanValue().booleanNot()) or result = evaluatesLen(expr, part, context, partval) - //or - //result = callable_test_evaluates_boolean(expr, use, context, val, origin) - //or - //result = hasattr_test_evaluates_boolean(expr, use, context, val, origin) + or + result = callable_test_evaluates(expr, part, context, partval) + or + result = hasattr_test_evaluates(expr, part, context, partval) ) } @@ -1067,6 +1067,49 @@ module Conditionals { result = TInt(val.(SequenceObjectInternal).length()) } + pragma [noinline] + private ObjectInternal callable_test_evaluates(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { + callable_call(call, use, context, val) and + ( + val = ObjectInternal::unknown() and result = ObjectInternal::bool(_) + or + val = ObjectInternal::unknownClass() and result = ObjectInternal::bool(_) + or + result = ObjectInternal::bool(Types::hasAttr(val.getClass(), "__call__")) + ) + } + + pragma [noinline] + private ObjectInternal hasattr_test_evaluates(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { + exists(string name | + hasattr_call(call, use, context, val, name) + | + val = ObjectInternal::unknown() and result = ObjectInternal::bool(_) + or + val = ObjectInternal::unknownClass() and result = ObjectInternal::bool(_) + or + result = ObjectInternal::bool(Types::hasAttr(val.getClass(), name)) + ) + } + + private predicate callable_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { + pinode_test_part(call, use) and + PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("callable"), _) and + use = call.getArg(0) and + PointsToInternal::pointsTo(use, context, val, _) + } + + private predicate hasattr_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, string name) { + pinode_test_part(call, use) and + PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("hasattr"), _) and + use = call.getArg(0) and + PointsToInternal::pointsTo(use, context, val, _) and + exists(StringObjectInternal str | + PointsToInternal::pointsTo(call.getArg(1), context, str, _) and + str.strValue() = name + ) + } + //private predicate isinstance_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls, ControlFlowNode origin) { pinode_test_part(call, use) and @@ -1129,6 +1172,16 @@ module Conditionals { ) } + predicate requireHasAttr(ClassObjectInternal cls, string name) { + cls != ObjectInternal::unknownClass() and + exists(ObjectInternal val | + val.getClass() = cls | + name = "__call__" and callable_call(_, _, _, val) + or + hasattr_call(_, _, _, val, name) + ) + } + pragma [noinline] private boolean inequalityEvaluatesBoolean(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val) { pinode_test_part(expr, use) and @@ -1433,5 +1486,27 @@ cached module Types { ) } + cached boolean hasAttr(ObjectInternal cls, string name) { + result = mroHasAttr(Types::getMro(cls), name, 0) + } + + private boolean mroHasAttr(ClassList mro, string name, int n) { + exists(ClassObjectInternal cls | + Conditionals::requireHasAttr(cls, name) and + mro = getMro(cls) + ) + and + ( + n = mro.length() and result = false + or + exists(ClassDecl decl | + decl = mro.getItem(n).getClassDeclaration() | + if decl.declaresAttribute(name) then + result = true + else + result = mroHasAttr(mro, name, n+1) + ) + ) + } } From 54c27e1d4b21cd368e06faac2c0e323bb53d59e9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 28 Mar 2019 16:43:35 +0000 Subject: [PATCH 025/128] Python points-to: Various minor performance tweaks. --- .../src/semmle/python/objects/Instances.qll | 31 ++++---- .../src/semmle/python/objects/Sequences.qll | 2 +- .../ql/src/semmle/python/objects/TObject.qll | 44 ++++++++---- .../src/semmle/python/pointsto/PointsTo.qll | 70 +++++++++++-------- 4 files changed, 92 insertions(+), 55 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index c12108e364ed..0f7b6624b7a5 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -197,12 +197,11 @@ class SelfInstanceInternal extends TSelfInstance, ObjectInternal { override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { - descriptor.isDescriptor() = true and - this = instance and - exists(AttrNode attr | - PointsToInternal::pointsTo(attr.getObject(name), _, this, _) and - this.getClass().attribute(name, descriptor, _) - ) + exists(AttrNode attr, ClassObjectInternal cls | + receiver_type(attr, name, this, cls) and + cls_descriptor(cls, name, descriptor) + ) and + instance = this } } @@ -288,17 +287,25 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { - descriptor.isDescriptor() = true and - this = instance and - exists(AttrNode attr | - PointsToInternal::pointsTo(attr.getObject(name), _, this, _) and - this.getClass().attribute(name, descriptor, _) - ) + exists(AttrNode attr, ClassObjectInternal cls | + receiver_type(attr, name, this, cls) and + cls_descriptor(cls, name, descriptor) + ) and + instance = this } } +private predicate receiver_type(AttrNode attr, string name, ObjectInternal value, ClassObjectInternal cls) { + PointsToInternal::pointsTo(attr.getObject(name), _, value, _) and value.getClass() = cls +} + +private predicate cls_descriptor(ClassObjectInternal cls, string name, ObjectInternal descriptor) { + cls.attribute(name, descriptor, _) and + descriptor.isDescriptor() = true +} + class SuperInstance extends TSuperInstance, ObjectInternal { override string toString() { diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index 00eeed8e5213..38c1c07c0c8a 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -36,7 +36,7 @@ abstract class SequenceObjectInternal extends ObjectInternal { abstract class TupleObjectInternal extends SequenceObjectInternal { override string toString() { - result = "tuple()" + result = "Tuple" } /** Gets the class declaration for this object, if it is a declared class. */ diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 80af6d93fbca..9216e9a798da 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -149,27 +149,43 @@ predicate literal_instantiation(ControlFlowNode n, ClassObjectInternal cls, Poin } predicate super_instantiation(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext context) { + super_2args(instantiation, self, startclass, context) + or + super_noargs(instantiation, self, startclass, context) +} + +pragma [noinline] +private predicate super_2args(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext context) { + exists(ControlFlowNode func, ControlFlowNode arg0, ControlFlowNode arg1 | + call2(instantiation, func, arg0, arg1) and + PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("super"), _) and + PointsToInternal::pointsTo(arg0, context, startclass, _) and + PointsToInternal::pointsTo(arg1, context, self, _) + ) +} + +pragma [noinline] +private predicate super_noargs(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext context) { PointsToInternal::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("super"), _) and - ( - PointsToInternal::pointsTo(instantiation.getArg(0), context, startclass, _) and - PointsToInternal::pointsTo(instantiation.getArg(1), context, self, _) - or - major_version() = 3 and - not exists(instantiation.getArg(0)) and - exists(Function func | - instantiation.getScope() = func and - /* Implicit class argument is lexically enclosing scope */ - func.getScope() = startclass.(PythonClassObjectInternal).getScope() and - /* Implicit 'self' is the 0th parameter */ - PointsToInternal::pointsTo(func.getArg(0).asName().getAFlowNode(), context, self, _) - ) + not exists(instantiation.getArg(0)) and + exists(Function func | + instantiation.getScope() = func and + /* Implicit class argument is lexically enclosing scope */ + func.getScope() = startclass.(PythonClassObjectInternal).getScope() and + /* Implicit 'self' is the 0th parameter */ + PointsToInternal::pointsTo(func.getArg(0).asName().getAFlowNode(), context, self, _) ) } +predicate call2(CallNode call, ControlFlowNode func, ControlFlowNode arg0, ControlFlowNode arg1) { + not exists(call.getArg(2)) and + func = call.getFunction() and + arg0 = call.getArg(0) and + arg1 = call.getArg(1) +} bindingset[self, function] predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext context) { - exists(ObjectInternal obj, string name | receiver(instantiation, context, obj, name) | exists(ObjectInternal cls | diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 35efc0edd7f0..056219342ac1 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -423,10 +423,11 @@ cached module PointsToInternal { /** Holds if ESSA edge refinement, `def`, refers to `(value, cls, origin)`. */ private predicate ssa_filter_definition_points_to(PyEdgeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - def.getSense() = Conditionals::testEvaluates(def.getTest(), def.getInput().getASourceUse(), context, value, origin) - //exists(ControlFlowNode test, ControlFlowNode use | - // refinement_test(test, use, Conditionals::branchEvaluatesTo(test, use, context, value, origin.toCfgNode()), def) - //) + def.getSense() = ssa_filter_definition_bool(def, context, value, origin) + } + + private boolean ssa_filter_definition_bool(PyEdgeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + result = Conditionals::testEvaluates(def.getTest(), def.getInput().getASourceUse(), context, value, origin) } /** Holds if ESSA definition, `uniphi`, refers to `(value, origin)`. */ @@ -982,6 +983,7 @@ module Conditionals { result = evaluates(expr, use, context, value, origin).booleanValue() } + pragma [noinline] ObjectInternal evaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { PointsToInternal::pointsTo(use, context, val, origin) and pinode_test(_, use) and expr = use and result = val @@ -1111,19 +1113,21 @@ module Conditionals { } //private - predicate isinstance_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls, ControlFlowNode origin) { + predicate isinstance_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls) { pinode_test_part(call, use) and - PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("isinstance"), _) and - use = call.getArg(0) and - PointsToInternal::pointsTo(use, context, val, origin) and - PointsToInternal::pointsTo(call.getArg(1), context, cls, _) + exists(ControlFlowNode func, ControlFlowNode arg1 | + call2(call, func, use, arg1) and + PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("isinstance"), _) and + PointsToInternal::pointsTo(use, context, val, _) and + PointsToInternal::pointsTo(arg1, context, cls, _) + ) } pragma [nomagic] //private boolean isinstance_test_evaluates_boolean(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { exists(ObjectInternal cls | - isinstance_call(call, use, context, val, cls, _) | + isinstance_call(call, use, context, val, cls) | result = Types::improperSubclass(val.getClass(), cls) or val = ObjectInternal::unknown() and result = maybe() @@ -1134,18 +1138,20 @@ module Conditionals { ) } - private predicate issubclass_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls, ControlFlowNode origin) { - PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("issubclass"), _) and - use = call.getArg(0) and - PointsToInternal::pointsTo(use, context, val, origin) and - PointsToInternal::pointsTo(call.getArg(1), context, cls, _) + private predicate issubclass_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls) { + pinode_test_part(call, use) and + exists(ControlFlowNode func, ControlFlowNode arg1 | + call2(call, func, use, arg1) and + PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("issubclass"), _) and + PointsToInternal::pointsTo(use, context, val, _) and + PointsToInternal::pointsTo(arg1, context, cls, _) + ) } pragma [nomagic] private boolean issubclass_test_evaluates_boolean(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { - pinode_test_part(call, use) and exists(ObjectInternal cls | - issubclass_call(call, use, context, val, cls, _) | + issubclass_call(call, use, context, val, cls) | result = Types::improperSubclass(val, cls) or val = ObjectInternal::unknownClass() and result = maybe() @@ -1161,14 +1167,17 @@ module Conditionals { predicate requireSubClass(ObjectInternal sub, ObjectInternal sup) { sup != ObjectInternal::unknownClass() and sub != ObjectInternal::unknownClass() and - (sup.isClass() = true or sup instanceof TupleObjectInternal) and - ( - issubclass_call(_, _, _, sub, sup, _) and sub.isClass() = true + exists(ObjectInternal sup_or_tuple | + issubclass_call(_, _, _, sub, sup_or_tuple) and sub.isClass() = true or exists(ObjectInternal val | - isinstance_call(_, _, _, val, sup, _) and + isinstance_call(_, _, _, val, sup_or_tuple) and sub = val.getClass() ) + | + sup = sup_or_tuple + or + sup = sup_or_tuple.(TupleObjectInternal).getItem(_) ) } @@ -1181,7 +1190,7 @@ module Conditionals { hasattr_call(_, _, _, val, name) ) } - + pragma [noinline] private boolean inequalityEvaluatesBoolean(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val) { pinode_test_part(expr, use) and @@ -1462,16 +1471,21 @@ cached module Types { cached boolean improperSubclass(ObjectInternal sub, ObjectInternal sup) { sub = sup and result = true or - result = mroContains(Types::getMro(sub), sup) - // TO DO... - // Handle tuples of classes + result = mroContains(Types::getMro(sub), sup, 0) + or + result = tupleSubclass(sub, sup, 0) } - private boolean mroContains(ClassList mro, ObjectInternal sup) { - result = mroContains(mro, sup, 0) + private boolean tupleSubclass(ObjectInternal cls, TupleObjectInternal tpl, int n) { + Conditionals::requireSubClass(cls, tpl) and + ( + n = tpl.length() and result = false + or + result = improperSubclass(cls, tpl.getItem(n)).booleanOr(tupleSubclass(cls, tpl, n+1)) + ) } - private boolean mroContains(ClassList mro, ObjectInternal sup, int n) { + private boolean mroContains(ClassList mro, ClassObjectInternal sup, int n) { exists(ClassObjectInternal cls | Conditionals::requireSubClass(cls, sup) and mro = getMro(cls) From ef0a6b67139b48b36ecdebe1f30a0749643b9c70 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 2 Apr 2019 14:15:24 +0100 Subject: [PATCH 026/128] Python points-to: Rationalize handling of expressions and conditions. Tweak API to be a bit more backward-compatible. --- .../src/semmle/python/objects/Callables.qll | 11 +- .../semmle/python/objects/ObjectInternal.qll | 5 + .../src/semmle/python/objects/Sequences.qll | 10 +- .../ql/src/semmle/python/objects/TObject.qll | 6 +- .../src/semmle/python/pointsto/PointsTo.qll | 412 ++++++++++++++---- .../ql/src/semmle/python/types/Builtins.qll | 11 +- .../PointsTo/new/TestEvaluate.ql | 7 +- 7 files changed, 355 insertions(+), 107 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 259496f702be..6dcd1dcb5042 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -156,14 +156,17 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } override predicate callResult(ObjectInternal obj, CfgOrigin origin) { - exists(Builtin func, ClassObjectInternal cls | + exists(Builtin func, BuiltinClassObjectInternal cls | func = this.getBuiltin() and func != Builtin::builtin("isinstance") and func != Builtin::builtin("issubclass") and - func != Builtin::builtin("callable") - | - cls = ObjectInternal::fromBuiltin(this.getReturnType()) and + func != Builtin::builtin("callable") and + cls = ObjectInternal::fromBuiltin(this.getReturnType()) | obj = TUnknownInstance(cls) + or + cls = ObjectInternal::noneType() and obj = ObjectInternal::none_() + or + cls = ObjectInternal::builtin("bool") and obj = ObjectInternal::bool(_) ) and origin = CfgOrigin::unknown() } diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 64726c89719c..ca5c6993627f 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -376,6 +376,11 @@ module ObjectInternal { ObjectInternal moduleType() { result = TBuiltinClassObject(Builtin::special("ModuleType")) } + + ObjectInternal noneType() { + result = TBuiltinClassObject(Builtin::special("NoneType")) + } + } /** Helper for boolean predicates returning both `true` and `false` */ diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index 38c1c07c0c8a..53ae9696dfde 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -36,7 +36,15 @@ abstract class SequenceObjectInternal extends ObjectInternal { abstract class TupleObjectInternal extends SequenceObjectInternal { override string toString() { - result = "Tuple" + this.length() = 0 and result = "()" + or + result = "(" + this.contents(0) + ")" + } + + string contents(int n) { + n = this.length() - 1 and result = this.getItem(n).toString() + or + result = this.getItem(n).toString() + ", " + this.contents(n+1) } /** Gets the class declaration for this object, if it is a declared class. */ diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 9216e9a798da..002af2719931 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -98,7 +98,11 @@ newtype TObject = function.isDescriptor() = true } or - TUnknownInstance(BuiltinClassObjectInternal cls) { cls != ObjectInternal::builtin("super") } + TUnknownInstance(BuiltinClassObjectInternal cls) { + cls != ObjectInternal::builtin("super") and + cls != ObjectInternal::builtin("bool") and + cls != ObjectInternal::noneType() + } or TSuperInstance(ObjectInternal self, ClassObjectInternal startclass) { super_instantiation(_, self, startclass, _) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 056219342ac1..bdb04ecd1757 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -118,6 +118,14 @@ module PointsTo { var.getDefinition().(ParameterDefinition).getDefiningNode() = f and ssa_variable_points_to(var, context, obj, cls, origin) ) + or + not f.isParameter() and + exists(Value value | + PointsToInternal::pointsTo(f.(DefinitionNode).getValue(), context, value, origin) and + cls = value.getClass().getSource() | + obj = value.getSource() or + not exists(value.getSource()) and obj = origin + ) } deprecated predicate @@ -151,18 +159,9 @@ cached module PointsToInternal { private predicate points_to_candidate(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { use_points_to(f, context, value, origin) or - /* Not necessary, but for backwards compatibility */ - def_points_to(f, context, value, origin) - or attribute_load_points_to(f, context, value, origin) or - subscript_points_to(f, context, value, origin) - or - binary_expr_points_to(f, context, value, origin) - or - origin = f and test_expr_points_to(f, context, value) - or - origin = f and unary_points_to(f, context, value) + Expressions::pointsTo(f, context, value, origin, _, _) or if_exp_points_to(f, context, value, origin) or @@ -242,12 +241,6 @@ cached module PointsToInternal { ) } - /** Gets an object pointed to by the definition of an ESSA variable. */ - pragma [noinline] - private predicate def_points_to(DefinitionNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - pointsTo(f.getValue(), context, value, origin) - } - pragma [noinline] private predicate use_points_to_maybe_origin(NameNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin_or_obj) { variablePointsTo(fast_local_variable(f), context, value, origin_or_obj) @@ -318,18 +311,7 @@ cached module PointsToInternal { /** Holds if `f` is an attribute `x.attr` and points to `(value, cls, origin)`. */ pragma [noinline] private predicate attribute_load_points_to(AttrNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - f.isLoad() and - exists(ObjectInternal object, string name | - pointsTo(f.getObject(name), context, object, _) - | - exists(CfgOrigin orig | - object.attribute(name, value, orig) and - origin = orig.fix(f) - ) - or - object.attributesUnknown() and - origin = f and value = ObjectInternal::unknown() - ) + none() // TO DO -- Support CustomPointsToAttribute //or //exists(CustomPointsToAttribute object, string name | @@ -525,44 +507,6 @@ cached module PointsToInternal { ) } - private predicate subscript_points_to(SubscriptNode sub, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - pointsTo(sub.getValue(), context, ObjectInternal::unknown(), _) and - value = ObjectInternal::unknown() and origin = sub - } - - /** Track bitwise expressions so we can handle integer flags and enums. - * Tracking too many binary expressions is likely to kill performance. - */ - private predicate binary_expr_points_to(BinaryExprNode b, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - // TO DO... - // Track some integer values through `|` and the types of some objects - none() - } - - pragma [noinline] - private predicate test_expr_points_to(ControlFlowNode cmp, PointsToContext context, ObjectInternal value) { - exists(ControlFlowNode use | - value = Conditionals::evaluates(cmp, use, context, _, _) and - use != cmp - ) - // or - // value = version_tuple_compare(cmp, context) - } - - pragma [noinline] - private predicate unary_points_to(UnaryExprNode f, PointsToContext context, ObjectInternal value) { - exists(Unaryop op, ObjectInternal operand | - op = f.getNode().getOp() and - pointsTo(f.getOperand(), context, operand, _) - | - op instanceof Not and value = ObjectInternal::bool(operand.booleanValue().booleanNot()) - or - op instanceof USub and value = ObjectInternal::fromInt(-operand.intValue()) - or - operand = ObjectInternal::unknown() and value = operand - ) - } - /** Holds if `f` is an expression node `tval if cond else fval` and points to `(value, origin)`. */ private predicate if_exp_points_to(IfExprNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { pointsTo(f.getAnOperand(), context, value, origin) @@ -976,48 +920,312 @@ private predicate potential_builtin_points_to(NameNode f, ObjectInternal value, ) } -module Conditionals { - boolean testEvaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - pinode_test(expr, use) and - result = evaluates(expr, use, context, value, origin).booleanValue() - } +module Expressions { - pragma [noinline] - ObjectInternal evaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { - PointsToInternal::pointsTo(use, context, val, origin) and - pinode_test(_, use) and expr = use and result = val - or - exists(ControlFlowNode part, ObjectInternal partval | - pinode_test_part(expr, part) and - partval = evaluates(part, use, context, val, origin) + predicate attributePointsTo(AttrNode attr, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode obj, ObjectInternal objvalue) { + attr.isLoad() and + exists(string name | + attr.getObject(name) = obj and + PointsTo::pointsTo(obj, context, objvalue, _) | - result = equalityEvaluates(expr, part, context, partval) + exists(CfgOrigin orig | + objvalue.attribute(name, value, orig) and + origin = orig.fix(attr) + ) or - result = ObjectInternal::bool(inequalityEvaluatesBoolean(expr, part, context, partval)) + objvalue.attributesUnknown() and + origin = attr and value = ObjectInternal::unknown() + ) + } + + predicate subscriptPointsTo(SubscriptNode subscr, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode obj, ObjectInternal objvalue) { + subscr.isLoad() and + obj = subscr.getObject() and + PointsTo::pointsTo(obj, context, objvalue, _) and + value = ObjectInternal::unknown() and origin = subscr + } + + /** Track bitwise expressions so we can handle integer flags and enums. + * Tracking too many binary expressions is likely to kill performance. + */ + predicate binaryPointsTo(BinaryExprNode b, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode operand, ObjectInternal opvalue) { + // TO DO... + // Track some integer values through `|` and the types of some objects + operand = b.getAnOperand() and + PointsTo::pointsTo(operand, context, opvalue, _) and + value = ObjectInternal::unknown() and origin = b + } + + predicate unaryPointsTo(UnaryExprNode u, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode operand, ObjectInternal opvalue) { + exists(Unaryop op | + op = u.getNode().getOp() and + operand = u.getOperand() and + PointsTo::pointsTo(operand, context, opvalue, _) + | + op instanceof Not and value = ObjectInternal::bool(opvalue.booleanValue().booleanNot()) or - result = ObjectInternal::bool(isinstance_test_evaluates_boolean(expr, part, context, partval)) + op instanceof USub and value = ObjectInternal::fromInt(-opvalue.intValue()) or - result = ObjectInternal::bool(issubclass_test_evaluates_boolean(expr, part, context, partval)) + opvalue = ObjectInternal::unknown() and value = opvalue + ) and + origin = u + } + + predicate builtinCallPointsTo(CallNode call, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode arg, ObjectInternal argvalue) { + PointsToInternal::pointsTo(arg, context, argvalue, _) and + arg = call.getArg(0) and + exists(BuiltinFunctionObjectInternal callable | + PointsToInternal::pointsTo(call.getFunction(), context, callable, _) | + callable = ObjectInternal::builtin("len") and value = TInt(argvalue.(SequenceObjectInternal).length()) or - exists(string attr | - expr.(AttrNode).getObject(attr) = use | - val.attribute(attr, result, _) + callable != ObjectInternal::builtin("len") and + callable != ObjectInternal::builtin("callable") and + callable != ObjectInternal::builtin("isinstance") and + callable != ObjectInternal::builtin("issubclass") and + callable.isClass() = false and + value = ObjectInternal::unknown() + ) and + origin = call + } + + private boolean otherComparisonEvaluatesTo(CompareNode comp, PointsToContext context, ControlFlowNode operand, ObjectInternal opvalue) { + exists(Cmpop op | + comp.operands(operand, op, _) and + (op instanceof In or op instanceof NotIn) | + PointsToInternal::pointsTo(operand, context, opvalue, _) + ) and result = maybe() + } + + pragma [noinline] + private boolean equalityEvaluatesTo(CompareNode comp, PointsToContext context, ControlFlowNode operand, ObjectInternal opvalue) { + exists(ControlFlowNode r, boolean sense | + equality_test(comp, operand, sense, r) and + exists(ObjectInternal other | + PointsToInternal::pointsTo(operand, context, opvalue, _) and + PointsToInternal::pointsTo(r, context, other, _) | + opvalue.isComparable() = true and other.isComparable() = true and + ( + other = opvalue and result = sense + or + other != opvalue and result = sense.booleanNot() + ) + or + opvalue.isComparable() = false and result = maybe() or - val.attributesUnknown() and result = ObjectInternal::unknown() + other.isComparable() = false and result = maybe() + ) + ) + } + + /** Holds if `c` is a test comparing `x` and `y`. `is` is true if the operator is `is` or `==`, it is false if the operator is `is not` or `!=`. */ + private predicate equality_test(CompareNode c, ControlFlowNode x, boolean is, ControlFlowNode y) { + exists(Cmpop op | + c.operands(x, op, y) or + c.operands(y, op, x) + | + (is = true and op instanceof Is or + is = false and op instanceof IsNot or + is = true and op instanceof Eq or + is = false and op instanceof NotEq ) + ) + } + + pragma [noinline] + private boolean inequalityEvaluatesTo(ControlFlowNode expr, PointsToContext context, ControlFlowNode use, ObjectInternal val) { + exists(ControlFlowNode r, boolean sense | + exists(boolean strict, ObjectInternal other | + ( + inequality(expr, use, r, strict) and sense = true + or + inequality(expr, r, use, strict) and sense = false + ) and + PointsToInternal::pointsTo(use, context, val, _) and + PointsToInternal::pointsTo(r, context, other, _) + | + val.intValue() < other.intValue() and result = sense + or + val.intValue() > other.intValue() and result = sense.booleanNot() + or + val.intValue() = other.intValue() and result = strict.booleanXor(sense) + or + val.strValue() < other.strValue() and result = sense + or + val.strValue() > other.strValue() and result = sense.booleanNot() + or + val.strValue() = other.strValue() and result = strict.booleanXor(sense) + or + val.isComparable() = false and result = maybe() + or + other.isComparable() = false and result = maybe() + ) + + ) + } + + /** Helper for comparisons. */ + private predicate inequality(CompareNode cmp, ControlFlowNode lesser, ControlFlowNode greater, boolean strict) { + exists(Cmpop op | + cmp.operands(lesser, op, greater) and op.getSymbol() = "<" and strict = true or - expr instanceof BinaryExprNode and result = ObjectInternal::unknown() + cmp.operands(lesser, op, greater) and op.getSymbol() = "<=" and strict = false or - part = not_operand(expr) and result = ObjectInternal::bool(partval.booleanValue().booleanNot()) + cmp.operands(greater, op, lesser) and op.getSymbol() = ">" and strict = true or - result = evaluatesLen(expr, part, context, partval) + cmp.operands(greater, op, lesser) and op.getSymbol() = ">=" and strict = false + ) + } + + predicate pointsTo(ControlFlowNode expr, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode subexpr, ObjectInternal subvalue) { + attributePointsTo(expr, context, value, origin, subexpr, subvalue) + or + subscriptPointsTo(expr, context, value, origin, subexpr, subvalue) + or + binaryPointsTo(expr, context, value, origin, subexpr, subvalue) + or + unaryPointsTo(expr, context, value, origin, subexpr, subvalue) + or + builtinCallPointsTo(expr, context, value, origin, subexpr, subvalue) + or + value = ObjectInternal::bool(evaluatesTo(expr, context, subexpr, subvalue)) and origin = expr + } + + boolean evaluatesTo(ControlFlowNode expr, PointsToContext context, ControlFlowNode subexpr, ObjectInternal subvalue) { + result = equalityEvaluatesTo(expr, context, subexpr, subvalue) + or + result = inequalityEvaluatesTo(expr, context, subexpr, subvalue) + or + result = otherComparisonEvaluatesTo(expr, context, subexpr, subvalue) + or + result = isinstanceEvaluatesTo(expr, context, subexpr, subvalue) + or + result = issubclassEvaluatesTo(expr, context, subexpr, subvalue) + or + result = callableEvaluatesTo(expr, context, subexpr, subvalue) + or + result = hasattrEvaluatesTo(expr, context, subexpr, subvalue) + } + + /** Holds if `expr` is the operand of a unary `not` expression. */ + private ControlFlowNode not_operand(ControlFlowNode expr) { + expr.(UnaryExprNode).getNode().getOp() instanceof Not and + result = expr.(UnaryExprNode).getOperand() + } + + pragma [nomagic] + //private + boolean isinstanceEvaluatesTo(CallNode call, PointsToContext context, ControlFlowNode use, ObjectInternal val) { + exists(ObjectInternal cls | + isinstance_call(call, use, context, val, cls) | + result = Types::improperSubclass(val.getClass(), cls) or - result = callable_test_evaluates(expr, part, context, partval) + val = ObjectInternal::unknown() and result = maybe() or - result = hasattr_test_evaluates(expr, part, context, partval) + cls = ObjectInternal::unknown() and result = maybe() + or + cls = ObjectInternal::unknownClass() and result = maybe() ) + } + //private + predicate isinstance_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls) { + exists(ControlFlowNode func, ControlFlowNode arg1 | + call2(call, func, use, arg1) and + PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("isinstance"), _) and + PointsToInternal::pointsTo(use, context, val, _) and + PointsToInternal::pointsTo(arg1, context, cls, _) + ) + } + + //private + predicate issubclass_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls) { + exists(ControlFlowNode func, ControlFlowNode arg1 | + call2(call, func, use, arg1) and + PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("issubclass"), _) and + PointsToInternal::pointsTo(use, context, val, _) and + PointsToInternal::pointsTo(arg1, context, cls, _) + ) + } + + pragma [nomagic] + private boolean issubclassEvaluatesTo(CallNode call, PointsToContext context, ControlFlowNode use, ObjectInternal val) { + exists(ObjectInternal cls | + issubclass_call(call, use, context, val, cls) | + result = Types::improperSubclass(val, cls) + or + val = ObjectInternal::unknownClass() and result = maybe() + or + val = ObjectInternal::unknown() and result = maybe() + or + cls = ObjectInternal::unknown() and result = maybe() + or + cls = ObjectInternal::unknownClass() and result = maybe() + ) + } + + pragma [noinline] + private boolean callableEvaluatesTo(CallNode call, PointsToContext context, ControlFlowNode use, ObjectInternal val) { + callable_call(call, use, context, val) and + ( + val = ObjectInternal::unknown() and result = maybe() + or + val = ObjectInternal::unknownClass() and result = maybe() + or + result = Types::hasAttr(val.getClass(), "__call__") + ) + } + + pragma [noinline] + private boolean hasattrEvaluatesTo(CallNode call, PointsToContext context, ControlFlowNode use, ObjectInternal val) { + exists(string name | + hasattr_call(call, use, context, val, name) + | + val = ObjectInternal::unknown() and result = maybe() + or + val = ObjectInternal::unknownClass() and result = maybe() + or + result = Types::hasAttr(val.getClass(), name) + ) + } + + private predicate callable_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { + PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("callable"), _) and + use = call.getArg(0) and + PointsToInternal::pointsTo(use, context, val, _) + } + + private predicate hasattr_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, string name) { + PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("hasattr"), _) and + use = call.getArg(0) and + PointsToInternal::pointsTo(use, context, val, _) and + exists(StringObjectInternal str | + PointsToInternal::pointsTo(call.getArg(1), context, str, _) and + str.strValue() = name + ) + } + + +} + + +module Conditionals { + + boolean testEvaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + pinode_test(expr, use) and + result = evaluates(expr, use, context, value, origin).booleanValue() + } + + pragma [noinline] + ObjectInternal evaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { + PointsToInternal::pointsTo(use, context, val, origin) and + pinode_test(_, use) and expr = use and result = val + or + exists(ControlFlowNode part, ObjectInternal partval | + pinode_test_part(expr, part) and + partval = evaluates(part, use, context, val, origin) and + Expressions::pointsTo(expr, context, result, _, part, partval) + ) } /** Holds if `expr` is the operand of a unary `not` expression. */ @@ -1062,11 +1270,22 @@ module Conditionals { } pragma [noinline] - private ObjectInternal evaluatesLen(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { + private ObjectInternal evaluatesCall(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { pinode_test_part(call, use) and PointsToInternal::pointsTo(use, context, val, _) and - PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("len"), _) and - result = TInt(val.(SequenceObjectInternal).length()) + exists(ObjectInternal callable | + PointsToInternal::pointsTo(call.getFunction(), context, callable, _) | + callable = ObjectInternal::builtin("len") and result = TInt(val.(SequenceObjectInternal).length()) + or + callable = TType() and result = val.getClass() + or + callable != ObjectInternal::builtin("len") and + callable != ObjectInternal::builtin("callable") and + callable != ObjectInternal::builtin("isinstance") and + callable != ObjectInternal::builtin("issubclass") and + callable != TType() and + result = ObjectInternal::unknown() + ) } pragma [noinline] @@ -1138,7 +1357,8 @@ module Conditionals { ) } - private predicate issubclass_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls) { + //private + predicate issubclass_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls) { pinode_test_part(call, use) and exists(ControlFlowNode func, ControlFlowNode arg1 | call2(call, func, use, arg1) and diff --git a/python/ql/src/semmle/python/types/Builtins.qll b/python/ql/src/semmle/python/types/Builtins.qll index 400ec15ab3e8..fc3108664ef8 100644 --- a/python/ql/src/semmle/python/types/Builtins.qll +++ b/python/ql/src/semmle/python/types/Builtins.qll @@ -51,7 +51,16 @@ class Builtin extends @py_cobject { } string getName() { - py_cobjectnames(this, result) + if this.isStr() then + result = "str" + else + py_cobjectnames(this, result) + } + + private predicate isStr() { + major_version() = 2 and this = Builtin::special("bytes") + or + major_version() = 3 and this = Builtin::special("unicode") } predicate isClass() { diff --git a/python/ql/test/library-tests/PointsTo/new/TestEvaluate.ql b/python/ql/test/library-tests/PointsTo/new/TestEvaluate.ql index 731b710d2c5e..2256d6910f13 100644 --- a/python/ql/test/library-tests/PointsTo/new/TestEvaluate.ql +++ b/python/ql/test/library-tests/PointsTo/new/TestEvaluate.ql @@ -5,10 +5,9 @@ import semmle.python.pointsto.PointsToContext import Util -from ControlFlowNode test, ControlFlowNode use, Object val, boolean eval, ClassObject cls, PointsToContext ctx +from ControlFlowNode test, ControlFlowNode use, Value val, boolean eval, PointsToContext ctx where not use instanceof NameConstantNode and not use.getNode() instanceof ImmutableLiteral and -PointsTo::points_to(use, ctx, val, cls, _) and -eval = PointsTo::test_evaluates_boolean(test, use, ctx, val, cls, _) -select locate(test.getLocation(), "bc"), test.getNode().toString(), eval.toString(), use.getNode().toString(), val.toString() +eval = Conditionals::testEvaluates(test, use, ctx, val, _) +select locate(test.getLocation(), "bc"), test.getNode().toString(), eval.toString(), use.getNode().toString(), val.getSource().toString() From 162bf5143ba5490b438a28eb4edbd817bd8a791c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 4 Apr 2019 10:02:46 +0100 Subject: [PATCH 027/128] Python points-to: Assorted improvements to performance and better compatibility. --- .../src/semmle/python/objects/Callables.qll | 5 +- .../ql/src/semmle/python/objects/Classes.qll | 7 +- .../ql/src/semmle/python/objects/Modules.qll | 36 +- .../src/semmle/python/objects/ObjectAPI.qll | 8 +- .../ql/src/semmle/python/pointsto/Context.qll | 2 +- .../src/semmle/python/pointsto/PointsTo.qll | 539 +++++++++--------- .../python/pointsto/PointsToContext.qll | 11 + 7 files changed, 307 insertions(+), 301 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 6dcd1dcb5042..8163aea99133 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -80,6 +80,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti this = TPythonFunctionObject(result) } + pragma [noinline] override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { exists(Function func, ControlFlowNode rval | func = this.getScope() and @@ -93,10 +94,11 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti ) } + pragma [noinline] override predicate callResult(ObjectInternal obj, CfgOrigin origin) { this.getScope().isProcedure() and obj = ObjectInternal::none_() and - origin = CfgOrigin::unknown() + origin = this.getScope().getEntryNode() } override predicate calleeAndOffset(Function scope, int paramOffset) { @@ -155,6 +157,7 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } + pragma [noinline] override predicate callResult(ObjectInternal obj, CfgOrigin origin) { exists(Builtin func, BuiltinClassObjectInternal cls | func = this.getBuiltin() and diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 896718fd6e64..e395e9bb4e8c 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -236,12 +236,7 @@ class TypeInternal extends ClassObjectInternal, TType { } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { - exists(CallNode call, PointsToContext caller, ObjectInternal instance | - callee.fromCall(call, caller) | - count(call.getAnArg()) = 1 and - PointsToInternal::pointsTo(call.getArg(0), caller, instance, origin) and - obj = instance.getClass() - ) + none() } override predicate callResult(ObjectInternal obj, CfgOrigin origin) { diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 471ed3ab0429..74e31a4796e3 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -23,10 +23,6 @@ abstract class ModuleObjectInternal extends ObjectInternal { none() } - override ControlFlowNode getOrigin() { - result = this.getSourceModule().getEntryNode() - } - override boolean isClass() { result = false } override boolean isComparable() { result = true } @@ -92,6 +88,10 @@ class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleOb override predicate attributesUnknown() { none() } + override ControlFlowNode getOrigin() { + none() + } + } class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { @@ -154,18 +154,14 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { this.getInitModule().attribute(name, value, origin) or - // TO DO, dollar variable... - //exists(Module init | - // init = this.getSourceModule() and - // not exists(EssaVariable var | var.getAUse() = init.getANormalExit() and var.getSourceVariable().getName() = name) and - // exists(EssaVariable var, Context context | - // isModuleStateVariable(var) and var.getAUse() = init.getANormalExit() and - // context.isImport() and - // SSA::ssa_variable_named_attribute_pointsTo(var, context, name, undefinedVariable(), _, origin) and - // value = this.submodule(name) - // ) - //) - //or + exists(Module init | + init = this.getSourceModule() and + not exists(EssaVariable var | var.getAUse() = init.getANormalExit() and var.getSourceVariable().getName() = name) and + ModuleAttributes::pointsToAtExit(init, name, ObjectInternal::undefined(), _) and + value = this.submodule(name) and + origin = CfgOrigin::fromModule(value) + ) + or this.hasNoInitModule() and exists(ModuleObjectInternal mod | mod = this.submodule(name) and @@ -176,6 +172,10 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { override predicate attributesUnknown() { none() } + override ControlFlowNode getOrigin() { + none() + } + } /** Get the ESSA pseudo-variable used to retain module state @@ -249,5 +249,9 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { override predicate attributesUnknown() { none() } + override ControlFlowNode getOrigin() { + result = this.getSourceModule().getEntryNode() + } + } diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 0d60f4b5f069..e0a099270219 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -53,8 +53,14 @@ class Value extends TObject { } /* For backwards compatibility with old API */ - ObjectSource getSource() { + deprecated ObjectSource getSource() { result = this.(ObjectInternal).getSource() + or + exists(Module p | + p.isPackage() and + p.getPath() = this.(PackageObjectInternal).getFolder() and + result = p.getEntryNode() + ) } /** Gets the `ControlFlowNode` that will be passed as the nth argument to `this` when called at `call`. diff --git a/python/ql/src/semmle/python/pointsto/Context.qll b/python/ql/src/semmle/python/pointsto/Context.qll index 156e8eb43b4a..57c047e8a671 100644 --- a/python/ql/src/semmle/python/pointsto/Context.qll +++ b/python/ql/src/semmle/python/pointsto/Context.qll @@ -1,4 +1,4 @@ import python private import semmle.python.pointsto.PointsToContext -class Context = PointsToContext; \ No newline at end of file +class Context = PointsToContext; diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index bdb04ecd1757..e4f86abbe66d 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -91,6 +91,12 @@ module CfgOrigin { result = mod.getSourceModule().getEntryNode() } + CfgOrigin fromObject(ObjectInternal obj) { + obj.isBuiltin() and result = unknown() + or + result = obj.getOrigin() + } + } /* The API */ @@ -108,15 +114,17 @@ module PointsTo { deprecated predicate points_to(ControlFlowNode f, PointsToContext context, Object obj, ClassObject cls, ControlFlowNode origin) { exists(Value value | - PointsToInternal::pointsTo(f, context, value, origin) and + pointsToValue(f, context, value, origin) and cls = value.getClass().getSource() | obj = value.getSource() or not exists(value.getSource()) and obj = origin ) or - f.isParameter() and exists(EssaVariable var | - var.getDefinition().(ParameterDefinition).getDefiningNode() = f and - ssa_variable_points_to(var, context, obj, cls, origin) + /* Backwards compatibility for *args and **kwargs */ + exists(Function func | + obj = f and origin = f and context.isRuntime() | + func.getVararg() = f.getNode() and cls = theTupleType() or + func.getKwarg() = f.getNode() and cls = theDictType() ) or not f.isParameter() and @@ -128,6 +136,14 @@ module PointsTo { ) } + private predicate pointsToValue(ControlFlowNode f, PointsToContext context, Value value, ControlFlowNode origin) { + PointsToInternal::pointsTo(f, context, value, origin) + or + exists(string name | + AttributePointsTo::attributePointsTo(f.(AttrNode).getObject(name), context, name, value, origin) + ) + } + deprecated predicate ssa_variable_points_to(EssaVariable var, PointsToContext context, Object obj, ClassObject cls, CfgOrigin origin) { exists(Value value | @@ -255,6 +271,33 @@ cached module PointsToInternal { pragma [noinline] cached predicate variablePointsTo(EssaVariable var, PointsToContext context, ObjectInternal value, CfgOrigin origin) { ssa_definition_points_to(var.getDefinition(), context, value, origin) + or + exists(EssaVariable prev | + ssaShortCut(prev, var) and + variablePointsTo(prev, context, value, origin) + ) + } + + private predicate ssaShortCut(EssaVariable start, EssaVariable end) { + end.getDefinition().(PhiFunction).getShortCircuitInput() = start + or + /* Attribute assignments have no effect as far as value tracking is concerned, except for `__class__`. */ + exists(AttributeAssignment def | + not def.getName() = "__class__" and + start = def.getInput() and + end.getDefinition() = def + ) + or + /* Ignore the effects of calls on their arguments. PointsTo is an approximation, + * but attempting to improve accuracy would be very expensive for very little gain. */ + exists(ArgumentRefinement def | + start = def.getInput() and + end.getDefinition() = def + ) + or + exists(EssaVariable mid | + ssaShortCut(start, mid) and ssaShortCut(mid, end) + ) } pragma [noinline] @@ -336,7 +379,7 @@ cached module PointsToInternal { pragma [noinline] private predicate ssa_node_definition_points_to(EssaNodeDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - reachableBlock(def.getDefiningNode().getBasicBlock(), _) and + reachableBlock(def.getDefiningNode().getBasicBlock(), context) and ssa_node_definition_points_to_unpruned(def, context, value, origin) } @@ -365,39 +408,42 @@ cached module PointsToInternal { pragma [noinline] private predicate ssa_node_refinement_points_to(EssaNodeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - //method_callsite_points_to(def, context, value, origin) - //or + method_callsite_points_to(def, context, value, origin) + or InterModulePointsTo::import_star_points_to(def, context, value, origin) or attribute_assignment_points_to(def, context, value, origin) or InterProceduralPointsTo::callsite_points_to(def, context, value, origin) or - argument_points_to(def, context, value, origin) - //or - //attribute_delete_points_to(def, context, value, origin) + attribute_delete_points_to(def, context, value, origin) or uni_edged_phi_points_to(def, context, value, origin) } - /** Attribute assignments have no effect as far as value tracking is concerned, except for `__class__`. */ - pragma [noinline] - private predicate attribute_assignment_points_to(AttributeAssignment def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - if def.getName() = "__class__" then - exists(ObjectInternal cls | - pointsTo(def.getValue(), context, cls, _) and - value = TUnknownInstance(cls) and - origin = CfgOrigin::fromCfgNode(def.getDefiningNode()) - ) - else - variablePointsTo(def.getInput(), context, value, origin) + /** Pass through for `self` for the implicit re-definition of `self` in `self.foo()`. */ + private predicate method_callsite_points_to(MethodCallsiteRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + /* The value of self remains the same, only the attributes may change */ + variablePointsTo(def.getInput(), context, value, origin) } - /** Ignore the effects of calls on their arguments. PointsTo is an approximation, but attempting to improve accuracy would be very expensive for very little gain. */ - private predicate argument_points_to(ArgumentRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + /** Attribute deletions have no effect as far as value tracking is concerned. */ + pragma [noinline] + private predicate attribute_delete_points_to(EssaAttributeDeletion def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { variablePointsTo(def.getInput(), context, value, origin) } + /** Attribute assignments have no effect as far as value tracking is concerned, except for `__class__`. */ + pragma [noinline] + private predicate attribute_assignment_points_to(AttributeAssignment def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + def.getName() = "__class__" and + exists(ObjectInternal cls | + pointsTo(def.getValue(), context, cls, _) and + value = TUnknownInstance(cls) and + origin = CfgOrigin::fromCfgNode(def.getDefiningNode()) + ) + } + private predicate self_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { origin = CfgOrigin::fromCfgNode(def.getDefiningNode()) and value.(SelfInstanceInternal).parameterAndContext(def, context) @@ -472,8 +518,6 @@ cached module PointsToInternal { or not exists(ConditionBlock guard | guard.controlsEdge(pred, phi.getBasicBlock(), _)) ) - or - variablePointsTo(phi.getShortCircuitInput(), context, value, origin) } /** Points-to for implicit variable declarations at scope-entry. */ @@ -665,7 +709,7 @@ module InterModulePointsTo { ) } - private boolean module_exports_boolean(ModuleObjectInternal mod, string name) { + boolean module_exports_boolean(ModuleObjectInternal mod, string name) { ofInterestInExports(mod, name) and exists(Module src | src = mod.getSourceModule() @@ -703,7 +747,7 @@ module InterProceduralPointsTo { pragma [noinline] predicate call_points_to(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(ObjectInternal func, CfgOrigin resultOrigin | - PointsToInternal::pointsTo(f.getFunction(), context, func, _) and + call_points_to_callee(f, context, func) and origin = resultOrigin.fix(f) | exists(PointsToContext callee | @@ -711,13 +755,38 @@ module InterProceduralPointsTo { func.callResult(callee, value, resultOrigin) ) or - func.callResult(value, resultOrigin) + func.callResult(value, resultOrigin) and + context.appliesTo(f) ) + or + call_to_type(f, context) and + exists(ObjectInternal arg | + PointsToInternal::pointsTo(f.getArg(0), context, arg, _) and + value = arg.getClass() | + value.isBuiltin() and origin = f + or + origin = value.getOrigin() + or + value = ObjectInternal::unknownClass() and origin = f + ) + } + + pragma [noinline] + private predicate call_to_type(CallNode f, PointsToContext context) { + count(f.getArg(_)) = 1 and + PointsToInternal::pointsTo(f.getFunction(), context, ObjectInternal::builtin("type"), _) + } + + pragma [noinline] + private predicate call_points_to_callee(CallNode f, PointsToContext context, ObjectInternal callee) { + PointsToInternal::pointsTo(f.getFunction(), context, callee, _) } /** Points-to for parameter. `def foo(param): ...`. */ pragma [noinline] predicate parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + self_parameter_points_to(def, context, value, origin) + or positional_parameter_points_to(def, context, value, origin) or named_parameter_points_to(def, context, value, origin) @@ -739,6 +808,17 @@ module InterProceduralPointsTo { context.isRuntime() and value = ObjectInternal::unknown() and origin = def.getDefiningNode() } + private predicate self_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + def.isSelf() and + exists(CallNode call, BoundMethodObjectInternal method, Function func, PointsToContext caller | + PointsToInternal::pointsTo(call.getFunction(), caller, method, _) and + context.fromCall(call, caller) and + func = method.getScope() and + def.getScope() = func and + value = method.getSelf() and + origin = CfgOrigin::fromObject(value) + ) + } /** Helper for `parameter_points_to` */ pragma [noinline] @@ -1107,12 +1187,6 @@ module Expressions { result = hasattrEvaluatesTo(expr, context, subexpr, subvalue) } - /** Holds if `expr` is the operand of a unary `not` expression. */ - private ControlFlowNode not_operand(ControlFlowNode expr) { - expr.(UnaryExprNode).getNode().getOp() instanceof Not and - result = expr.(UnaryExprNode).getOperand() - } - pragma [nomagic] //private boolean isinstanceEvaluatesTo(CallNode call, PointsToContext context, ControlFlowNode use, ObjectInternal val) { @@ -1128,65 +1202,34 @@ module Expressions { ) } - //private + private predicate isinstance_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls) { exists(ControlFlowNode func, ControlFlowNode arg1 | call2(call, func, use, arg1) and - PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("isinstance"), _) and + points_to_isinstance(func, context) and PointsToInternal::pointsTo(use, context, val, _) and PointsToInternal::pointsTo(arg1, context, cls, _) ) } - //private + private predicate issubclass_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls) { exists(ControlFlowNode func, ControlFlowNode arg1 | call2(call, func, use, arg1) and - PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("issubclass"), _) and + points_to_issubclass(func, context) and PointsToInternal::pointsTo(use, context, val, _) and PointsToInternal::pointsTo(arg1, context, cls, _) ) } - pragma [nomagic] - private boolean issubclassEvaluatesTo(CallNode call, PointsToContext context, ControlFlowNode use, ObjectInternal val) { - exists(ObjectInternal cls | - issubclass_call(call, use, context, val, cls) | - result = Types::improperSubclass(val, cls) - or - val = ObjectInternal::unknownClass() and result = maybe() - or - val = ObjectInternal::unknown() and result = maybe() - or - cls = ObjectInternal::unknown() and result = maybe() - or - cls = ObjectInternal::unknownClass() and result = maybe() - ) + pragma[noinline] + private predicate points_to_isinstance(ControlFlowNode func, PointsToContext context) { + PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("isinstance"), _) } - pragma [noinline] - private boolean callableEvaluatesTo(CallNode call, PointsToContext context, ControlFlowNode use, ObjectInternal val) { - callable_call(call, use, context, val) and - ( - val = ObjectInternal::unknown() and result = maybe() - or - val = ObjectInternal::unknownClass() and result = maybe() - or - result = Types::hasAttr(val.getClass(), "__call__") - ) - } - - pragma [noinline] - private boolean hasattrEvaluatesTo(CallNode call, PointsToContext context, ControlFlowNode use, ObjectInternal val) { - exists(string name | - hasattr_call(call, use, context, val, name) - | - val = ObjectInternal::unknown() and result = maybe() - or - val = ObjectInternal::unknownClass() and result = maybe() - or - result = Types::hasAttr(val.getClass(), name) - ) + pragma[noinline] + private predicate points_to_issubclass(ControlFlowNode func, PointsToContext context) { + PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("issubclass"), _) } private predicate callable_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { @@ -1196,191 +1239,60 @@ module Expressions { } private predicate hasattr_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, string name) { - PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("hasattr"), _) and - use = call.getArg(0) and - PointsToInternal::pointsTo(use, context, val, _) and - exists(StringObjectInternal str | - PointsToInternal::pointsTo(call.getArg(1), context, str, _) and - str.strValue() = name - ) - } - - -} - - -module Conditionals { - - boolean testEvaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - pinode_test(expr, use) and - result = evaluates(expr, use, context, value, origin).booleanValue() - } - - pragma [noinline] - ObjectInternal evaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { - PointsToInternal::pointsTo(use, context, val, origin) and - pinode_test(_, use) and expr = use and result = val - or - exists(ControlFlowNode part, ObjectInternal partval | - pinode_test_part(expr, part) and - partval = evaluates(part, use, context, val, origin) and - Expressions::pointsTo(expr, context, result, _, part, partval) - ) - } - - /** Holds if `expr` is the operand of a unary `not` expression. */ - private ControlFlowNode not_operand(ControlFlowNode expr) { - expr.(UnaryExprNode).getNode().getOp() instanceof Not and - result = expr.(UnaryExprNode).getOperand() - } - - pragma [noinline] - private ObjectInternal equalityEvaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val) { - exists(ControlFlowNode r, boolean sense | - pinode_test_part(expr, use) and equality_test(expr, use, sense, r) and - exists(ObjectInternal other | - PointsToInternal::pointsTo(use, context, val, _) and - PointsToInternal::pointsTo(r, context, other, _) | - val.isComparable() = true and other.isComparable() = true and - ( - other = val and result = ObjectInternal::bool(sense) - or - other != val and result = ObjectInternal::bool(sense.booleanNot()) - ) - or - val.isComparable() = false and result = ObjectInternal::bool(_) - or - other.isComparable() = false and result = ObjectInternal::bool(_) + exists(ControlFlowNode func, ControlFlowNode arg1 | + call2(call, func, use, arg1) and + points_to_hasattr(func, context) and + PointsToInternal::pointsTo(use, context, val, _) and + exists(StringObjectInternal str | + PointsToInternal::pointsTo(arg1, context, str, _) and + str.strValue() = name ) ) } - /** Holds if `c` is a test comparing `x` and `y`. `is` is true if the operator is `is` or `==`, it is false if the operator is `is not` or `!=`. */ - private predicate equality_test(CompareNode c, ControlFlowNode x, boolean is, ControlFlowNode y) { - exists(Cmpop op | - c.operands(x, op, y) or - c.operands(y, op, x) - | - (is = true and op instanceof Is or - is = false and op instanceof IsNot or - is = true and op instanceof Eq or - is = false and op instanceof NotEq - ) - ) + pragma[noinline] + private predicate points_to_hasattr(ControlFlowNode func, PointsToContext context) { + PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("hasattr"), _) } - pragma [noinline] - private ObjectInternal evaluatesCall(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { - pinode_test_part(call, use) and - PointsToInternal::pointsTo(use, context, val, _) and - exists(ObjectInternal callable | - PointsToInternal::pointsTo(call.getFunction(), context, callable, _) | - callable = ObjectInternal::builtin("len") and result = TInt(val.(SequenceObjectInternal).length()) + pragma [nomagic] + private boolean issubclassEvaluatesTo(CallNode call, PointsToContext context, ControlFlowNode use, ObjectInternal val) { + exists(ObjectInternal cls | + issubclass_call(call, use, context, val, cls) | + result = Types::improperSubclass(val, cls) + or + val = ObjectInternal::unknownClass() and result = maybe() + or + val = ObjectInternal::unknown() and result = maybe() or - callable = TType() and result = val.getClass() + cls = ObjectInternal::unknown() and result = maybe() or - callable != ObjectInternal::builtin("len") and - callable != ObjectInternal::builtin("callable") and - callable != ObjectInternal::builtin("isinstance") and - callable != ObjectInternal::builtin("issubclass") and - callable != TType() and - result = ObjectInternal::unknown() + cls = ObjectInternal::unknownClass() and result = maybe() ) } pragma [noinline] - private ObjectInternal callable_test_evaluates(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { + private boolean callableEvaluatesTo(CallNode call, PointsToContext context, ControlFlowNode use, ObjectInternal val) { callable_call(call, use, context, val) and ( - val = ObjectInternal::unknown() and result = ObjectInternal::bool(_) + val = ObjectInternal::unknown() and result = maybe() or - val = ObjectInternal::unknownClass() and result = ObjectInternal::bool(_) + val = ObjectInternal::unknownClass() and result = maybe() or - result = ObjectInternal::bool(Types::hasAttr(val.getClass(), "__call__")) + result = Types::hasAttr(val.getClass(), "__call__") ) } pragma [noinline] - private ObjectInternal hasattr_test_evaluates(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { + private boolean hasattrEvaluatesTo(CallNode call, PointsToContext context, ControlFlowNode use, ObjectInternal val) { exists(string name | hasattr_call(call, use, context, val, name) | - val = ObjectInternal::unknown() and result = ObjectInternal::bool(_) - or - val = ObjectInternal::unknownClass() and result = ObjectInternal::bool(_) - or - result = ObjectInternal::bool(Types::hasAttr(val.getClass(), name)) - ) - } - - private predicate callable_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { - pinode_test_part(call, use) and - PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("callable"), _) and - use = call.getArg(0) and - PointsToInternal::pointsTo(use, context, val, _) - } - - private predicate hasattr_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, string name) { - pinode_test_part(call, use) and - PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("hasattr"), _) and - use = call.getArg(0) and - PointsToInternal::pointsTo(use, context, val, _) and - exists(StringObjectInternal str | - PointsToInternal::pointsTo(call.getArg(1), context, str, _) and - str.strValue() = name - ) - } - - //private - predicate isinstance_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls) { - pinode_test_part(call, use) and - exists(ControlFlowNode func, ControlFlowNode arg1 | - call2(call, func, use, arg1) and - PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("isinstance"), _) and - PointsToInternal::pointsTo(use, context, val, _) and - PointsToInternal::pointsTo(arg1, context, cls, _) - ) - } - - pragma [nomagic] - //private - boolean isinstance_test_evaluates_boolean(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { - exists(ObjectInternal cls | - isinstance_call(call, use, context, val, cls) | - result = Types::improperSubclass(val.getClass(), cls) - or val = ObjectInternal::unknown() and result = maybe() or - cls = ObjectInternal::unknown() and result = maybe() - or - cls = ObjectInternal::unknownClass() and result = maybe() - ) - } - - //private - predicate issubclass_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, ObjectInternal cls) { - pinode_test_part(call, use) and - exists(ControlFlowNode func, ControlFlowNode arg1 | - call2(call, func, use, arg1) and - PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("issubclass"), _) and - PointsToInternal::pointsTo(use, context, val, _) and - PointsToInternal::pointsTo(arg1, context, cls, _) - ) - } - - pragma [nomagic] - private boolean issubclass_test_evaluates_boolean(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { - exists(ObjectInternal cls | - issubclass_call(call, use, context, val, cls) | - result = Types::improperSubclass(val, cls) - or val = ObjectInternal::unknownClass() and result = maybe() or - val = ObjectInternal::unknown() and result = maybe() - or - cls = ObjectInternal::unknown() and result = maybe() - or - cls = ObjectInternal::unknownClass() and result = maybe() + result = Types::hasAttr(val.getClass(), name) ) } @@ -1411,49 +1323,25 @@ module Conditionals { ) } - pragma [noinline] - private boolean inequalityEvaluatesBoolean(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val) { - pinode_test_part(expr, use) and - exists(ControlFlowNode r, boolean sense | - exists(boolean strict, ObjectInternal other | - ( - inequality(expr, use, r, strict) and sense = true - or - inequality(expr, r, use, strict) and sense = false - ) and - PointsToInternal::pointsTo(use, context, val, _) and - PointsToInternal::pointsTo(r, context, other, _) - | - val.intValue() < other.intValue() and result = sense - or - val.intValue() > other.intValue() and result = sense.booleanNot() - or - val.intValue() = other.intValue() and result = strict.booleanXor(sense) - or - val.strValue() < other.strValue() and result = sense - or - val.strValue() > other.strValue() and result = sense.booleanNot() - or - val.strValue() = other.strValue() and result = strict.booleanXor(sense) - or - val.isComparable() = false and result = maybe() - or - other.isComparable() = false and result = maybe() - ) +} - ) + +module Conditionals { + + boolean testEvaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + pinode_test(expr, use) and + result = evaluates(expr, use, context, value, origin).booleanValue() } - /** Helper for comparisons. */ - private predicate inequality(CompareNode cmp, ControlFlowNode lesser, ControlFlowNode greater, boolean strict) { - exists(Cmpop op | - cmp.operands(lesser, op, greater) and op.getSymbol() = "<" and strict = true - or - cmp.operands(lesser, op, greater) and op.getSymbol() = "<=" and strict = false - or - cmp.operands(greater, op, lesser) and op.getSymbol() = ">" and strict = true - or - cmp.operands(greater, op, lesser) and op.getSymbol() = ">=" and strict = false + pragma [noinline] + ObjectInternal evaluates(ControlFlowNode expr, ControlFlowNode use, PointsToContext context, ObjectInternal val, ControlFlowNode origin) { + PointsToInternal::pointsTo(use, context, val, origin) and + pinode_test(_, use) and expr = use and result = val + or + exists(ControlFlowNode part, ObjectInternal partval | + pinode_test_part(expr, part) and + partval = evaluates(part, use, context, val, origin) and + Expressions::pointsTo(expr, context, result, _, part, partval) ) } @@ -1697,7 +1585,7 @@ cached module Types { } private boolean tupleSubclass(ObjectInternal cls, TupleObjectInternal tpl, int n) { - Conditionals::requireSubClass(cls, tpl) and + Expressions::requireSubClass(cls, tpl) and ( n = tpl.length() and result = false or @@ -1707,7 +1595,7 @@ cached module Types { private boolean mroContains(ClassList mro, ClassObjectInternal sup, int n) { exists(ClassObjectInternal cls | - Conditionals::requireSubClass(cls, sup) and + Expressions::requireSubClass(cls, sup) and mro = getMro(cls) ) and @@ -1726,7 +1614,7 @@ cached module Types { private boolean mroHasAttr(ClassList mro, string name, int n) { exists(ClassObjectInternal cls | - Conditionals::requireHasAttr(cls, name) and + Expressions::requireHasAttr(cls, name) and mro = getMro(cls) ) and @@ -1744,3 +1632,102 @@ cached module Types { } } + + +module AttributePointsTo { + + predicate attributePointsTo(ControlFlowNode f, Context context, string name, ObjectInternal value, CfgOrigin origin) { + exists(ObjectInternal obj, Context prev, AttributeAssignment def | + PointsToInternal::pointsTo(f, context, obj, _) and + PointsToInternal::variablePointsTo(def.getInput(), prev, obj, _) and + PointsToInternal::pointsTo(def.getValue(), prev, value, origin) and name = def.getName() + | + prev.getOuter*().getCall().getBasicBlock().reaches(context.getOuter*().getCall().getBasicBlock()) + or + def.getScope().getScope*().precedes(f.getScope().getScope*()) + ) + } + +} + +module ModuleAttributes { + + private EssaVariable varAtExit(Module mod, string name) { + result.getName() = name and result.getAUse() = mod.getANormalExit() + } + + EssaVariable moduleStateVariable(ControlFlowNode use) { + result.getName() = "$" and result.getAUse() = use + } + + private EssaVariable moduleStateVarAtExit(Module mod) { + result = moduleStateVariable(mod.getANormalExit()) + } + + predicate pointsToAtExit(Module mod, string name, ObjectInternal value, CfgOrigin origin) { + if exists(varAtExit(mod, name)) then ( + PointsToInternal::variablePointsTo(varAtExit(mod, name), any(Context c | c.isImport()), value, origin) + ) else ( + moduleAttributePointsTo(moduleStateVarAtExit(mod), name, value, origin) + ) + } + + private predicate moduleAttributePointsTo(EssaVariable var, string name, ObjectInternal value, CfgOrigin origin) { + importStarPointsTo(var.getDefinition(), name, value, origin) + or + callsitePointsTo(var.getDefinition(), name, value, origin) + or + scopeEntryPointsTo(var.getDefinition(), name, value, origin) + } + + pragma [noinline] + private predicate importStarPointsTo(ImportStarRefinement def, string name, ObjectInternal value, CfgOrigin origin) { + def.getVariable().getName() = "$" and + exists(ImportStarNode imp, ModuleObjectInternal mod, CfgOrigin orig | + origin = orig.fix(imp) and + PointsToInternal::pointsTo(def.getDefiningNode().(ImportStarNode).getModule(), any(Context ctx | ctx.isImport()), mod, _) | + /* Attribute from imported module */ + InterModulePointsTo::module_exports_boolean(mod, name) = true and + mod.attribute(name, value, orig) and + not exists(Variable v | v.getId() = name and v.getScope() = imp.getScope()) + or + /* Retain value held before import */ + InterModulePointsTo::module_exports_boolean(mod, name) = false and + moduleAttributePointsTo(def.getInput().getDefinition(), name, value, origin) + ) + } + + /** Points-to for a variable (possibly) redefined by a call: + * `var = ...; foo(); use(var)` + * Where var may be redefined in call to `foo` if `var` escapes (is global or non-local). + */ + pragma [noinline] + predicate callsitePointsTo(CallsiteRefinement def, string name, ObjectInternal value, CfgOrigin origin) { + def.getVariable().getName() = "$" and + exists(EssaVariable var, Function func, PointsToContext callee | + InterProceduralPointsTo::callsite_calls_function(def.getCall(), _, func, callee, _) and + var = moduleStateVariable(func.getANormalExit()) and + moduleAttributePointsTo(var, name, value, origin) + ) + } + + pragma [noinline] + predicate scopeEntryPointsTo(ScopeEntryDefinition def, string name, ObjectInternal value, CfgOrigin origin) { + def.getVariable().getName() = "$" and + /* Transfer from another scope */ + exists(EssaVariable var, PointsToContext outer | + InterProceduralPointsTo::scope_entry_value_transfer(var, outer, def, _) and + moduleAttributePointsTo(var, name, value, origin) + ) + or + def.getVariable().getName() = "$" and + exists(PackageObjectInternal package | + package.getSourceModule() = def.getScope() and + exists(package.submodule(name)) and + value = ObjectInternal::undefined() and + origin = CfgOrigin::unknown() + ) + } + +} + diff --git a/python/ql/src/semmle/python/pointsto/PointsToContext.qll b/python/ql/src/semmle/python/pointsto/PointsToContext.qll index 2f9f34c81fb1..c57ff45504a2 100644 --- a/python/ql/src/semmle/python/pointsto/PointsToContext.qll +++ b/python/ql/src/semmle/python/pointsto/PointsToContext.qll @@ -119,6 +119,17 @@ private cached newtype TPointsToContext = total_cost(call, outerContext) = cost and cost <= max_context_cost() } + or + TObjectContext(SelfInstanceInternal object) + +module Context { + + PointsToContext forObject(ObjectInternal object) { + result = TObjectContext(object) + } + +} + /** Points-to context. Context can be one of: * * "main": Used for scripts. From 31a95ceeec9df8f8b1f60b2a8c06ff5d6d25a7cf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 4 Apr 2019 10:51:28 +0100 Subject: [PATCH 028/128] Python points-to: Use strongly typed version of CfgOrigin. --- .../src/semmle/python/objects/Callables.qll | 7 +- .../ql/src/semmle/python/objects/Classes.qll | 2 +- .../ql/src/semmle/python/objects/Modules.qll | 6 +- .../semmle/python/objects/ObjectInternal.qll | 4 +- .../src/semmle/python/pointsto/PointsTo.qll | 151 ++++++++++-------- .../src/semmle/python/types/ClassObject.qll | 7 +- .../src/semmle/python/types/ModuleObject.qll | 7 +- 7 files changed, 100 insertions(+), 84 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 8163aea99133..971bdc8294af 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -82,11 +82,12 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti pragma [noinline] override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { - exists(Function func, ControlFlowNode rval | + exists(Function func, ControlFlowNode rval, ControlFlowNode forigin | func = this.getScope() and callee.appliesToScope(func) | rval = func.getAReturnValueFlowNode() and - PointsToInternal::pointsTo(rval, callee, obj, origin) + PointsToInternal::pointsTo(rval, callee, obj, forigin) and + origin = CfgOrigin::fromCfgNode(forigin) or PointsToInternal::reachableBlock(blockReturningNone(func), callee) and obj = ObjectInternal::none_() and @@ -98,7 +99,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti override predicate callResult(ObjectInternal obj, CfgOrigin origin) { this.getScope().isProcedure() and obj = ObjectInternal::none_() and - origin = this.getScope().getEntryNode() + origin = CfgOrigin::fromCfgNode(this.getScope().getEntryNode()) } override predicate calleeAndOffset(Function scope, int paramOffset) { diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index e395e9bb4e8c..61a36618919f 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -143,7 +143,7 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec } override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - value = ObjectInternal::fromBuiltin(this.getBuiltin().getMember(name)) and + value = ObjectInternal::fromBuiltin(this.getBuiltin().getMember(name)) and origin = CfgOrigin::unknown() } diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 74e31a4796e3..d2d5ee188d48 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -159,21 +159,21 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { not exists(EssaVariable var | var.getAUse() = init.getANormalExit() and var.getSourceVariable().getName() = name) and ModuleAttributes::pointsToAtExit(init, name, ObjectInternal::undefined(), _) and value = this.submodule(name) and - origin = CfgOrigin::fromModule(value) + origin = CfgOrigin::fromObject(value) ) or this.hasNoInitModule() and exists(ModuleObjectInternal mod | mod = this.submodule(name) and value = mod | - origin = CfgOrigin::fromModule(mod) + origin = CfgOrigin::fromObject(mod) ) } override predicate attributesUnknown() { none() } override ControlFlowNode getOrigin() { - none() + result = this.getSourceModule().getEntryNode() } } diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index ca5c6993627f..3073d64ca4b9 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -77,9 +77,9 @@ class ObjectInternal extends TObject { abstract predicate attributesUnknown(); - /** For backwards compatibility shim -- Not all objects have a "source" + /** For backwards compatibility shim -- Not all objects have a "source". * Objects (except unknown and undefined values) should attempt to return - * exactly one result for either this method`. + * exactly one result for this method. * */ @py_object getSource() { result = this.getOrigin() diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index e4f86abbe66d..d783c725dea2 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -8,44 +8,7 @@ private import semmle.python.pointsto.MRO2 private import semmle.python.types.Builtins /* Use this version for speed */ -library class CfgOrigin extends @py_object { - - string toString() { - /* Not to be displayed */ - none() - } - - /** Get a `ControlFlowNode` from `this` or `here`. - * If `this` is a ControlFlowNode then use that, otherwise fall back on `here` - */ - pragma[inline] - ControlFlowNode asCfgNodeOrHere(ControlFlowNode here) { - result = this - or - not this instanceof ControlFlowNode and result = here - } - - ControlFlowNode toCfgNode() { - result = this - } - - pragma[inline] - CfgOrigin fix(ControlFlowNode here) { - if this = Builtin::unknown() then - result = here - else - result = this - } - -} - -/* Use this version for stronger type-checking */ -//private newtype TCfgOrigin = -// TUnknownOrigin() -// or -// TCfgOrigin(ControlFlowNode f) -// -//library class CfgOrigin extends TCfgOrigin { +//library class CfgOrigin extends @py_object { // // string toString() { // /* Not to be displayed */ @@ -57,44 +20,92 @@ library class CfgOrigin extends @py_object { // */ // pragma[inline] // ControlFlowNode asCfgNodeOrHere(ControlFlowNode here) { -// this = TUnknownOrigin() and result = here +// result = this // or -// this = TCfgOrigin(result) +// not this instanceof ControlFlowNode and result = here // } // // ControlFlowNode toCfgNode() { -// this = TCfgOrigin(result) +// result = this // } // +// pragma[inline] // CfgOrigin fix(ControlFlowNode here) { -// this = TUnknownOrigin() and result = TCfgOrigin(here) -// or -// not this = TUnknownOrigin() and result = this +// if this = Builtin::unknown() then +// result = here +// else +// result = this // } +// //} // +//module CfgOrigin { +// +// CfgOrigin fromCfgNode(ControlFlowNode f) { +// result = f +// } +// +// CfgOrigin unknown() { +// result = Builtin::unknown() +// } +// +// CfgOrigin fromObject(ObjectInternal obj) { +// obj.isBuiltin() and result = unknown() +// or +// result = obj.getOrigin() +// } +// +//} +/* Use this version for stronger type-checking */ +private newtype TCfgOrigin = + TUnknownOrigin() + or + TFlowNodeOrigin(ControlFlowNode f) -module CfgOrigin { +library class CfgOrigin extends TCfgOrigin { - CfgOrigin fromCfgNode(ControlFlowNode f) { - result = f + string toString() { + /* Not to be displayed */ + none() } - CfgOrigin unknown() { - result = Builtin::unknown() + /** Get a `ControlFlowNode` from `this` or `here`. + * If `this` is a ControlFlowNode then use that, otherwise fall back on `here` + */ + pragma[inline] + ControlFlowNode asCfgNodeOrHere(ControlFlowNode here) { + this = TUnknownOrigin() and result = here + or + this = TFlowNodeOrigin(result) } - CfgOrigin fromModule(ModuleObjectInternal mod) { - mod.isBuiltin() and result = unknown() + ControlFlowNode toCfgNode() { + this = TFlowNodeOrigin(result) + } + + CfgOrigin fix(ControlFlowNode here) { + this = TUnknownOrigin() and result = TFlowNodeOrigin(here) or - result = mod.getSourceModule().getEntryNode() + not this = TUnknownOrigin() and result = this + } + +} + +module CfgOrigin { + + CfgOrigin fromCfgNode(ControlFlowNode f) { + result = TFlowNodeOrigin(f) + } + + CfgOrigin unknown() { + result = TUnknownOrigin() } CfgOrigin fromObject(ObjectInternal obj) { obj.isBuiltin() and result = unknown() or - result = obj.getOrigin() + result = fromCfgNode(obj.getOrigin()) } } @@ -106,7 +117,7 @@ module PointsTo { PointsToInternal::pointsTo(f, context, value, origin) } - predicate variablePointsTo(EssaVariable var, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + predicate variablePointsTo(EssaVariable var, PointsToContext context, ObjectInternal value, CfgOrigin origin) { PointsToInternal::variablePointsTo(var, context, value, origin) } @@ -149,8 +160,7 @@ module PointsTo { exists(Value value | PointsToInternal::variablePointsTo(var, context, value, origin) and cls = value.getClass().getSource() | - obj = value.getSource() or - not exists(value.getSource()) and obj = origin + obj = value.getSource() ) } @@ -444,17 +454,20 @@ cached module PointsToInternal { ) } - private predicate self_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - origin = CfgOrigin::fromCfgNode(def.getDefiningNode()) and + private predicate self_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + origin = def.getDefiningNode() and value.(SelfInstanceInternal).parameterAndContext(def, context) } /** Holds if ESSA edge refinement, `def`, refers to `(value, cls, origin)`. */ private predicate ssa_filter_definition_points_to(PyEdgeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - def.getSense() = ssa_filter_definition_bool(def, context, value, origin) + exists(ControlFlowNode orig | + def.getSense() = ssa_filter_definition_bool(def, context, value, orig) and + origin = CfgOrigin::fromCfgNode(orig) + ) } - private boolean ssa_filter_definition_bool(PyEdgeRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + private boolean ssa_filter_definition_bool(PyEdgeRefinement def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { result = Conditionals::testEvaluates(def.getTest(), def.getInput().getASourceUse(), context, value, origin) } @@ -643,7 +656,7 @@ module InterModulePointsTo { exists(PackageObjectInternal package | package.getSourceModule() = def.getDefiningNode().getScope() | value = package.submodule(def.getSourceVariable().getName()) and - origin = CfgOrigin::fromModule(value).fix(def.getDefiningNode()) and + origin = CfgOrigin::fromObject(value).asCfgNodeOrHere(def.getDefiningNode()) and context.isImport() ) } @@ -748,7 +761,7 @@ module InterProceduralPointsTo { predicate call_points_to(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(ObjectInternal func, CfgOrigin resultOrigin | call_points_to_callee(f, context, func) and - origin = resultOrigin.fix(f) + origin = resultOrigin.asCfgNodeOrHere(f) | exists(PointsToContext callee | callee.fromCall(f, context) and @@ -816,7 +829,7 @@ module InterProceduralPointsTo { func = method.getScope() and def.getScope() = func and value = method.getSelf() and - origin = CfgOrigin::fromObject(value) + origin = value.getOrigin() ) } @@ -1007,11 +1020,11 @@ module Expressions { attr.isLoad() and exists(string name | attr.getObject(name) = obj and - PointsTo::pointsTo(obj, context, objvalue, _) + PointsToInternal::pointsTo(obj, context, objvalue, _) | exists(CfgOrigin orig | objvalue.attribute(name, value, orig) and - origin = orig.fix(attr) + origin = orig.asCfgNodeOrHere(attr) ) or objvalue.attributesUnknown() and @@ -1022,7 +1035,7 @@ module Expressions { predicate subscriptPointsTo(SubscriptNode subscr, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode obj, ObjectInternal objvalue) { subscr.isLoad() and obj = subscr.getObject() and - PointsTo::pointsTo(obj, context, objvalue, _) and + PointsToInternal::pointsTo(obj, context, objvalue, _) and value = ObjectInternal::unknown() and origin = subscr } @@ -1033,7 +1046,7 @@ module Expressions { // TO DO... // Track some integer values through `|` and the types of some objects operand = b.getAnOperand() and - PointsTo::pointsTo(operand, context, opvalue, _) and + PointsToInternal::pointsTo(operand, context, opvalue, _) and value = ObjectInternal::unknown() and origin = b } @@ -1041,7 +1054,7 @@ module Expressions { exists(Unaryop op | op = u.getNode().getOp() and operand = u.getOperand() and - PointsTo::pointsTo(operand, context, opvalue, _) + PointsToInternal::pointsTo(operand, context, opvalue, _) | op instanceof Not and value = ObjectInternal::bool(opvalue.booleanValue().booleanNot()) or @@ -1636,7 +1649,7 @@ cached module Types { module AttributePointsTo { - predicate attributePointsTo(ControlFlowNode f, Context context, string name, ObjectInternal value, CfgOrigin origin) { + predicate attributePointsTo(ControlFlowNode f, Context context, string name, ObjectInternal value, ControlFlowNode origin) { exists(ObjectInternal obj, Context prev, AttributeAssignment def | PointsToInternal::pointsTo(f, context, obj, _) and PointsToInternal::variablePointsTo(def.getInput(), prev, obj, _) and diff --git a/python/ql/src/semmle/python/types/ClassObject.qll b/python/ql/src/semmle/python/types/ClassObject.qll index ed34eff58142..bd34f5f39949 100644 --- a/python/ql/src/semmle/python/types/ClassObject.qll +++ b/python/ql/src/semmle/python/types/ClassObject.qll @@ -134,10 +134,11 @@ class ClassObject extends Object { /** Whether the named attribute refers to the object, class and origin */ predicate attributeRefersTo(string name, Object obj, ClassObject cls, ControlFlowNode origin) { - exists(Value val | - theClass().attribute(name, val, origin) and + exists(Value val, CfgOrigin valorig | + theClass().attribute(name, val, valorig) and obj = val.getSource() and - cls = val.getClass().getSource() + cls = val.getClass().getSource() and + origin = valorig.toCfgNode() ) } diff --git a/python/ql/src/semmle/python/types/ModuleObject.qll b/python/ql/src/semmle/python/types/ModuleObject.qll index 78376c35d4e6..7519218703bd 100644 --- a/python/ql/src/semmle/python/types/ModuleObject.qll +++ b/python/ql/src/semmle/python/types/ModuleObject.qll @@ -50,9 +50,10 @@ abstract class ModuleObject extends Object { } predicate attributeRefersTo(string name, Object obj, ControlFlowNode origin) { - exists(Value val | - theModule().(ModuleObjectInternal).attribute(name, val, origin) and - obj = val.getSource() + exists(Value val, CfgOrigin valorig | + theModule().(ModuleObjectInternal).attribute(name, val, valorig) and + obj = val.getSource() and + origin = valorig.toCfgNode() ) } From 3c3048084578c30c31d3de7f9c17ce4a7c5c67bf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 5 Apr 2019 08:59:58 +0100 Subject: [PATCH 029/128] Python: Extend API a bit. --- .../ql/src/semmle/python/objects/ObjectAPI.qll | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index e0a099270219..1ba5295cc86d 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -104,6 +104,18 @@ module Module { } +module Value { + + Value named(string name) { + exists(string modname, string attrname | + name = modname + "." + attrname | + result = Module::named(modname).attr(attrname) + ) + or + result = ObjectInternal::builtin(name) + } + +} class CallableValue extends Value { @@ -124,5 +136,10 @@ class ClassValue extends Value { this.(ObjectInternal).isClass() = true } + /** Gets an improper super type of this class. */ + ClassValue getASuperType() { + result = Types::getMro(this).getAnItem() + } + } From 782311f805fd57c6370f2c34ee94d1b75a27f6de Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 5 Apr 2019 10:40:53 +0100 Subject: [PATCH 030/128] Python: Update taint-tracking to use new points-to API. --- .../src/semmle/python/objects/ObjectAPI.qll | 8 ++ .../semmle/python/security/TaintTracking.qll | 78 ++++++++++--------- .../semmle/python/security/strings/Basic.qll | 4 +- .../src/semmle/python/web/pyramid/Request.qll | 4 +- .../src/semmle/python/web/webob/Request.qll | 4 +- 5 files changed, 56 insertions(+), 42 deletions(-) diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 1ba5295cc86d..1aaf9c4e25db 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -94,6 +94,10 @@ class ModuleValue extends Value { result = this.(ModuleObjectInternal).getName() } + Module getScope() { + result = this.(ModuleObjectInternal).getSourceModule() + } + } module Module { @@ -128,6 +132,10 @@ class CallableValue extends Value { none() } + Function getScope() { + result = this.(PythonFunctionObjectInternal).getScope() + } + } class ClassValue extends Value { diff --git a/python/ql/src/semmle/python/security/TaintTracking.qll b/python/ql/src/semmle/python/security/TaintTracking.qll index d3212c5b5486..9d076b2558fd 100755 --- a/python/ql/src/semmle/python/security/TaintTracking.qll +++ b/python/ql/src/semmle/python/security/TaintTracking.qll @@ -133,12 +133,17 @@ abstract class TaintKind extends string { ) } + /** DEPRECATED -- Use getType() instead */ + ClassObject getClass() { + none() + } + /** Gets the class of this kind of taint. * For example, if this were a kind of string taint * the `result` would be `theStrType()`. */ - ClassObject getClass() { - none() + ClassValue getType() { + result.getSource() = this.getClass() } /** Gets the boolean values (may be one, neither, or both) that @@ -194,7 +199,7 @@ class SequenceKind extends CollectionKind { mod.getOp() instanceof Mod and mod.getAnOperand() = fromnode and result = this.getItem() and - result.getClass() = theStrType() + result.getType() = Value::named("str") ) } @@ -279,7 +284,7 @@ module DictKind { predicate flowStep(ControlFlowNode fromnode, ControlFlowNode tonode) { TaintFlowImplementation::copyCall(fromnode, tonode) or - tonode.(CallNode).getFunction().refersTo(theDictType()) and + tonode.(CallNode).getFunction().pointsTo(Value::named("dict")) and tonode.(CallNode).getArg(0) = fromnode } @@ -947,8 +952,8 @@ library module TaintFlowImplementation { pragma [noinline] private predicate import_flow(TaintedNode fromnode, ImportExprNode tonode, CallContext tocontext, string name) { - exists(ModuleObject mod | - tonode.refersTo(mod) and + exists(ModuleValue mod | + tonode.pointsTo(mod) and module_attribute_tainted(mod, name, fromnode) and tocontext.appliesTo(tonode) ) @@ -965,9 +970,9 @@ library module TaintFlowImplementation { pragma [noinline] predicate from_import_step(TaintedNode fromnode, TrackedValue totaint, CallContext tocontext, ControlFlowNode tonode) { - exists(string name, ImportExprNode fmod, ModuleObject mod | + exists(string name, ImportExprNode fmod, ModuleValue mod | fmod = tonode.(ImportMemberNode).getModule(name) and - fmod.refersTo(mod) and + fmod.pointsTo(mod) and tocontext.appliesTo(tonode) and module_attribute_tainted(mod, name, fromnode) and totaint = fromnode.getTrackedValue() @@ -977,7 +982,7 @@ library module TaintFlowImplementation { pragma [noinline] predicate getattr_step(TaintedNode fromnode, TrackedValue totaint, CallContext tocontext, CallNode tonode) { exists(ControlFlowNode arg, string name | - tonode.getFunction().refersTo(Object::builtin("getattr")) and + tonode.getFunction().pointsTo(Value::named("getattr")) and arg = tonode.getArg(0) and name = tonode.getArg(1).getNode().(StrConst).getText() and arg = fromnode.getNode() and @@ -1017,11 +1022,11 @@ library module TaintFlowImplementation { ) } - predicate module_attribute_tainted(ModuleObject m, string name, TaintedNode origin) { + predicate module_attribute_tainted(ModuleValue m, string name, TaintedNode origin) { exists(EssaVariable var, CallContext c | var.getName() = name and BaseFlow::reaches_exit(var) and - var.getScope() = m.getModule() and + var.getScope() = m.getScope() and tainted_var(var, c, origin) and c = TTop() ) @@ -1062,9 +1067,9 @@ library module TaintFlowImplementation { } predicate self_init_end_transfer(EssaVariable self, CallContext callee, CallNode call, CallContext caller) { - exists(ClassObject cls, Function init | - PointsTo::instantiation(call, _, cls) and - init = cls.lookupAttribute("__init__").(FunctionObject).getFunction() and + exists(ClassValue cls, Function init | + call.getFunction().pointsTo(cls) and + init = cls.attr("__init__").(CallableValue).getScope() and self.getSourceVariable().(Variable).isSelf() and self.getScope() = init | callee = caller.getCallee(call) @@ -1198,10 +1203,10 @@ library module TaintFlowImplementation { predicate parameter_step(CallContext caller, ControlFlowNode argument, CallContext callee, NameNode param) { exists(ParameterDefinition def | def.getDefiningNode() = param and - exists(FunctionObject func, CallNode call | - exists(int n | argument = func.getArgumentForCall(call, n) and param.getNode() = func.getFunction().getArg(n)) + exists(CallableValue func, CallNode call | + exists(int n | argument = func.getArgumentForCall(call, n) and param.getNode() = func.getScope().getArg(n)) or - exists(string name | argument = func.getNamedArgumentForCall(call, name) and param.getNode() = func.getFunction().getArgByName(name)) + exists(string name | argument = func.getNamedArgumentForCall(call, name) and param.getNode() = func.getScope().getArgByName(name)) or class_initializer_argument(_, _, call, func, argument, param) | @@ -1211,11 +1216,11 @@ library module TaintFlowImplementation { } pragma [noinline] - predicate class_initializer_argument(ClassObject cls, int n, CallNode call, FunctionObject func, ControlFlowNode argument, NameNode param) { - PointsTo::instantiation(call, _, cls) and - cls.lookupAttribute("__init__") = func and + predicate class_initializer_argument(ClassValue cls, int n, CallNode call, CallableValue func, ControlFlowNode argument, NameNode param) { + call.getFunction().pointsTo(cls) and + cls.attr("__init__") = func and call.getArg(n) = argument and - param.getNode() = func.getFunction().getArg(n+1) + param.getNode() = func.getScope().getArg(n+1) } pragma [noinline] @@ -1257,15 +1262,15 @@ library module TaintFlowImplementation { not Filters::isinstance(test.getTest(), _, var.getSourceVariable().getAUse()) and not boolean_filter(test.getTest(), var.getSourceVariable().getAUse()) or - exists(ControlFlowNode c, ClassObject cls | + exists(ControlFlowNode c, ClassValue cls | Filters::isinstance(test.getTest(), c, var.getSourceVariable().getAUse()) - and c.refersTo(cls) + and c.pointsTo(cls) | test.getSense() = true and not exists(kind.getClass()) or - test.getSense() = true and kind.getClass().getAnImproperSuperType() = cls + test.getSense() = true and kind.getClass().getASuperType() = cls or - test.getSense() = false and not kind.getClass().getAnImproperSuperType() = cls + test.getSense() = false and not kind.getType().getASuperType() = cls ) or test.getSense() = test_evaluates(test.getTest(), var.getSourceVariable().getAUse(), kind) @@ -1311,8 +1316,9 @@ library module TaintFlowImplementation { pragma [noinline] predicate tainted_import_star(ImportStarRefinement def, CallContext context, TaintedNode origin) { - exists(ModuleObject mod, string name | - PointsTo::Flow::module_and_name_for_import_star(mod, name, def, _) | + exists(ModuleValue mod, string name | + PointsTo::pointsTo(def.getDefiningNode().(ImportStarNode).getModule(), _, mod, _) and + name = def.getSourceVariable().getName() | if mod.exports(name) then ( /* Attribute from imported module */ module_attribute_tainted(mod, name, origin) and @@ -1362,7 +1368,7 @@ library module TaintFlowImplementation { tonode.getArg(0) = fromnode ) or - tonode.getFunction().refersTo(Object::builtin("reversed")) and + tonode.getFunction().pointsTo(Value::named("reversed")) and tonode.getArg(0) = fromnode } @@ -1501,11 +1507,11 @@ class CallContext extends TCallContext { f.getFunction() = s and f.getACall() = call ) or - exists(ClassObject cls,CallNode call | + exists(ClassValue cls,CallNode call | this = TCalleeContext(call, _, _) and - PointsTo::instantiation(call, _, cls) and - s = cls.lookupAttribute("__init__").(FunctionObject).getFunction() and - call.getFunction().refersTo(cls) + call.getFunction().pointsTo(cls) and + s = cls.attr("__init__").(CallableValue).getScope() and + call.getFunction().pointsTo(cls) ) } @@ -1625,7 +1631,7 @@ pragma [noinline] private predicate dict_construct(ControlFlowNode itemnode, ControlFlowNode dictnode) { dictnode.(DictNode).getAValue() = itemnode or - dictnode.(CallNode).getFunction().refersTo(theDictType()) and + dictnode.(CallNode).getFunction().pointsTo(Value::named("dict")) and dictnode.(CallNode).getArgByName(_) = itemnode } @@ -1648,11 +1654,11 @@ private predicate sequence_call(ControlFlowNode fromnode, CallNode tonode) { tonode.getArg(0) = fromnode and exists(ControlFlowNode cls | cls = tonode.getFunction() | - cls.refersTo(theListType()) + cls.pointsTo(Value::named("list")) or - cls.refersTo(theTupleType()) + cls.pointsTo(Value::named("tuple")) or - cls.refersTo(theSetType()) + cls.pointsTo(Value::named("set")) ) } diff --git a/python/ql/src/semmle/python/security/strings/Basic.qll b/python/ql/src/semmle/python/security/strings/Basic.qll index 443377d6a64c..5e92543ec086 100755 --- a/python/ql/src/semmle/python/security/strings/Basic.qll +++ b/python/ql/src/semmle/python/security/strings/Basic.qll @@ -27,8 +27,8 @@ abstract class StringKind extends TaintKind { result = this and copy_call(fromnode, tonode) } - override ClassObject getClass() { - result = theStrType() or result = theUnicodeType() + override ClassValue getType() { + result = Value::named("bytes") or result = Value::named("str") or result = Value::named("unicode") } } diff --git a/python/ql/src/semmle/python/web/pyramid/Request.qll b/python/ql/src/semmle/python/web/pyramid/Request.qll index 7f9556bc4f9e..53750b3cb7c2 100644 --- a/python/ql/src/semmle/python/web/pyramid/Request.qll +++ b/python/ql/src/semmle/python/web/pyramid/Request.qll @@ -11,8 +11,8 @@ class PyramidRequest extends BaseWebobRequest { this = "pyramid.request" } - override ClassObject getClass() { - result = ModuleObject::named("pyramid.request").attr("Request") + override ClassValue getType() { + result = Value::named("pyramid.request.Request") } } diff --git a/python/ql/src/semmle/python/web/webob/Request.qll b/python/ql/src/semmle/python/web/webob/Request.qll index 8a82888ac5e8..1662313e0b93 100644 --- a/python/ql/src/semmle/python/web/webob/Request.qll +++ b/python/ql/src/semmle/python/web/webob/Request.qll @@ -44,8 +44,8 @@ class WebobRequest extends BaseWebobRequest { this = "webob.Request" } - override ClassObject getClass() { - result = ModuleObject::named("webob.request").attr("Request") + override ClassValue getType() { + result = Value::named("webob.request.Request") } } From fc2c46fe4adca34e3ce25d1e05a1c571d8ed495b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 5 Apr 2019 10:55:04 +0100 Subject: [PATCH 031/128] Python: Fix error in update Module to use new points-to API. --- python/ql/src/semmle/python/Module.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ql/src/semmle/python/Module.qll b/python/ql/src/semmle/python/Module.qll index ae9fa6c87e5c..9b6b8129e7ad 100644 --- a/python/ql/src/semmle/python/Module.qll +++ b/python/ql/src/semmle/python/Module.qll @@ -69,7 +69,7 @@ class Module extends Module_, Scope, AstNode { or exists(ModuleValue mod | mod.getSource() = this.getEntryNode() | - not mod.exports(result) + mod.exports(result) ) } From 662aedcb133a47d04a2518f5442b12ecc2d4593b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 8 Apr 2019 13:08:01 +0100 Subject: [PATCH 032/128] Python points-to: Fix up module attributes and classmethods. --- python/ql/src/semmle/python/Flow.qll | 22 +++----- .../src/semmle/python/objects/Callables.qll | 44 ++++++++++------ .../ql/src/semmle/python/objects/Modules.qll | 25 +++++---- .../src/semmle/python/objects/ObjectAPI.qll | 6 --- .../semmle/python/objects/ObjectInternal.qll | 2 + .../ql/src/semmle/python/objects/TObject.qll | 8 +-- .../src/semmle/python/pointsto/PointsTo.qll | 52 +++++++++++-------- .../ql/src/semmle/python/types/Builtins.qll | 3 -- .../src/semmle/python/types/ModuleObject.qll | 9 ++++ python/ql/src/semmle/python/types/Object.qll | 17 ++++++ .../objects/classes/Test.expected | 1 - .../library-tests/objects/classes/Test.ql | 9 ---- .../library-tests/objects/classes/test.py | 8 --- 13 files changed, 112 insertions(+), 94 deletions(-) delete mode 100644 python/ql/test/library-tests/objects/classes/Test.expected delete mode 100644 python/ql/test/library-tests/objects/classes/Test.ql delete mode 100644 python/ql/test/library-tests/objects/classes/test.py diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll index 94d4942620a3..fcec979f9070 100755 --- a/python/ql/src/semmle/python/Flow.qll +++ b/python/ql/src/semmle/python/Flow.qll @@ -217,6 +217,11 @@ class ControlFlowNode extends @py_flow_node { this.pointsTo(_, value, _) } + /** Gets the value that this ControlFlowNode points-to. */ + Value pointsTo() { + this.pointsTo(_, result, _) + } + /** The value and origin that this ControlFlowNode points-to. */ predicate pointsTo(Value value, ControlFlowNode origin) { this.pointsTo(_, value, origin) @@ -239,14 +244,7 @@ class ControlFlowNode extends @py_flow_node { /** Gets what this expression might "refer-to" in the given `context`. */ predicate refersTo(Context context, Object obj, ClassObject cls, ControlFlowNode origin) { - exists(Value value | - PointsTo::pointsTo(this, context, value, origin) and - cls = value.getClass().getSource() | - if exists(value.getSource().(Object)) then - obj = value.getSource() - else - obj = origin - ) + PointsTo::points_to(this, context, obj, cls, origin) } /** Whether this flow node might "refer-to" to `value` which is from `origin` @@ -254,13 +252,7 @@ class ControlFlowNode extends @py_flow_node { * where the class cannot be inferred. */ predicate refersTo(Object obj, ControlFlowNode origin) { - exists(Value value | - PointsTo::pointsTo(this, _, value, origin) | - if exists(value.getSource().(Object)) then - obj = value.getSource() - else - obj = origin - ) + PointsTo::points_to(this, _, obj, _, origin) } /** Equivalent to `this.refersTo(value, _)` */ diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 971bdc8294af..f870a0e7fd39 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -80,7 +80,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti this = TPythonFunctionObject(result) } - pragma [noinline] + pragma [nomagic] override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { exists(Function func, ControlFlowNode rval, ControlFlowNode forigin | func = this.getScope() and @@ -88,7 +88,15 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti rval = func.getAReturnValueFlowNode() and PointsToInternal::pointsTo(rval, callee, obj, forigin) and origin = CfgOrigin::fromCfgNode(forigin) - or + ) + or + procedureReturnsNone(callee, obj, origin) + } + + private predicate procedureReturnsNone(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { + exists(Function func | + func = this.getScope() and + callee.appliesToScope(func) | PointsToInternal::reachableBlock(blockReturningNone(func), callee) and obj = ObjectInternal::none_() and origin = CfgOrigin::unknown() @@ -349,7 +357,7 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { override string toString() { - result = "classmethod()" + result = "classmethod(" + this.getFunction() + ")" } override boolean booleanValue() { result = true } @@ -369,7 +377,7 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { override boolean isClass() { result = false } - override ObjectInternal getClass() { result = ObjectInternal::builtin("classmethod") } + override ObjectInternal getClass() { result = ObjectInternal::classMethod() } override boolean isComparable() { none() } @@ -385,9 +393,7 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { override string strValue() { none() } - override predicate calleeAndOffset(Function scope, int paramOffset) { - this.getFunction().calleeAndOffset(scope, paramOffset) - } + override predicate calleeAndOffset(Function scope, int paramOffset) { none() } override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } @@ -397,17 +403,25 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { any(ObjectInternal obj).binds(instance, _, this) and - ( - instance.isClass() = false and - value = TBoundMethod(instance.getClass(), this.getFunction()) + exists(ObjectInternal cls | + instance.isClass() = false and cls = instance.getClass() or - instance.isClass() = true and - value = TBoundMethod(instance, this.getFunction()) - ) and - origin = CfgOrigin::unknown() + instance.isClass() = true and cls = instance + | + value = TBoundMethod(cls, this.getFunction()) and + origin = CfgOrigin::unknown() + ) } - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + override predicate binds(ObjectInternal cls, string name, ObjectInternal descriptor) { + descriptor = this.getFunction() and + exists(ObjectInternal instance | + any(ObjectInternal obj).binds(instance, name, this) | + instance.isClass() = false and cls = instance.getClass() + or + instance.isClass() = true and cls = instance + ) + } } diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index d2d5ee188d48..29d5ea5ecf5c 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -156,8 +156,11 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { or exists(Module init | init = this.getSourceModule() and - not exists(EssaVariable var | var.getAUse() = init.getANormalExit() and var.getSourceVariable().getName() = name) and - ModuleAttributes::pointsToAtExit(init, name, ObjectInternal::undefined(), _) and + ( + not exists(EssaVariable var | var.getAUse() = init.getANormalExit() and var.getSourceVariable().getName() = name) + or + ModuleAttributes::pointsToAtExit(init, name, ObjectInternal::undefined(), _) + ) and value = this.submodule(name) and origin = CfgOrigin::fromObject(value) ) @@ -176,6 +179,14 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { result = this.getSourceModule().getEntryNode() } + override @py_object getSource() { + exists(Module package | + package.isPackage() and + package.getPath() = this.getFolder() and + result = package.getEntryNode() + ) + } + } /** Get the ESSA pseudo-variable used to retain module state @@ -237,14 +248,8 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { imp.isImport() and value != ObjectInternal::undefined() ) - // TO DO, dollar variable... - //or - //not exists(EssaVariable var | var.getAUse() = m.getANormalExit() and var.getSourceVariable().getName() = name) and - //exists(EssaVariable var, PointsToContext imp | - // var.getAUse() = m.getANormalExit() and isModuleStateVariable(var) | - // PointsToInternal::ssa_variable_named_attribute_pointsTo(var, imp, name, obj, origin) and - // imp.isImport() and obj != ObjectInternal::undefined() - //) + or + ModuleAttributes::pointsToAtExit(this.getSourceModule(), name, value, origin) } override predicate attributesUnknown() { none() } diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 1aaf9c4e25db..3a3742329f05 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -55,12 +55,6 @@ class Value extends TObject { /* For backwards compatibility with old API */ deprecated ObjectSource getSource() { result = this.(ObjectInternal).getSource() - or - exists(Module p | - p.isPackage() and - p.getPath() = this.(PackageObjectInternal).getFolder() and - result = p.getEntryNode() - ) } /** Gets the `ControlFlowNode` that will be passed as the nth argument to `this` when called at `call`. diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 3073d64ca4b9..8089cacb6556 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -363,6 +363,8 @@ module ObjectInternal { result = TBuiltinOpaqueObject(b) or result = TBuiltinModuleObject(b) + or + result = TBuiltinMethodObject(b) } ObjectInternal classMethod() { diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 002af2719931..033db340449c 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -6,7 +6,7 @@ private import semmle.python.pointsto.PointsToContext newtype TObject = TBuiltinClassObject(Builtin bltn) { - bltn.isClass() and + bltn.isClass() and not bltn = Builtin::unknownType() and not bltn = Builtin::special("type") } @@ -137,7 +137,7 @@ predicate static_method(CallNode instantiation, CallableObjectInternal function, } predicate class_method(CallNode instantiation, CallableObjectInternal function, PointsToContext context) { - PointsToInternal::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("classmethod"), _) and + PointsToInternal::pointsTo(instantiation.getFunction(), context, ObjectInternal::classMethod(), _) and PointsToInternal::pointsTo(instantiation.getArg(0), context, function, _) } @@ -321,8 +321,8 @@ library class ClassDecl extends @py_object { name = "unicode" or name = "tuple" or name = "property" or - name = "classmethod" or - name = "staticmethod" or + name = "ClassMethod" or + name = "StaticMethod" or name = "MethodType" or name = "ModuleType" ) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index d783c725dea2..74769a4ce6c8 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -84,6 +84,7 @@ library class CfgOrigin extends TCfgOrigin { this = TFlowNodeOrigin(result) } + pragma[inline] CfgOrigin fix(ControlFlowNode here) { this = TUnknownOrigin() and result = TFlowNodeOrigin(here) or @@ -128,7 +129,7 @@ module PointsTo { pointsToValue(f, context, value, origin) and cls = value.getClass().getSource() | obj = value.getSource() or - not exists(value.getSource()) and obj = origin + not exists(value.getSource()) and obj = origin and not cls = theBoundMethodType() ) or /* Backwards compatibility for *args and **kwargs */ @@ -663,22 +664,20 @@ module InterModulePointsTo { /** Points-to for `from ... import *`. */ predicate import_star_points_to(ImportStarRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - exists(CfgOrigin orig | - origin = orig.fix(def.getDefiningNode()) - | - exists(ModuleObjectInternal mod, string name | - PointsToInternal::pointsTo(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) and - name = def.getSourceVariable().getName() | - /* Attribute from imported module */ - module_exports_boolean(mod, name) = true and - mod.attribute(name, value, origin) - ) - or - exists(EssaVariable var | - /* Retain value held before import */ - variable_not_redefined_by_import_star(var, context, def) and - PointsToInternal::variablePointsTo(var, context, value,orig) - ) + /* Attribute from imported module */ + exists(CfgOrigin orig, ImportStarNode imp, ModuleObjectInternal mod, string name | + imp = def.getDefiningNode() and + PointsToInternal::pointsTo(imp.getModule(), context, mod, _) and + name = def.getSourceVariable().getName() and + module_exports_boolean(mod, name) = true and + mod.attribute(name, value, orig) and + origin = orig.fix(imp) + ) + or + /* Retain value held before import */ + exists(EssaVariable var | + variable_not_redefined_by_import_star(var, context, def) and + PointsToInternal::variablePointsTo(var, context, value, origin) ) } @@ -751,6 +750,8 @@ module InterModulePointsTo { ) or name = "__name__" and result = true + or + exists(mod.(BuiltinModuleObjectInternal).getBuiltin().getMember(name)) and result = true } } @@ -1650,14 +1651,18 @@ cached module Types { module AttributePointsTo { predicate attributePointsTo(ControlFlowNode f, Context context, string name, ObjectInternal value, ControlFlowNode origin) { - exists(ObjectInternal obj, Context prev, AttributeAssignment def | - PointsToInternal::pointsTo(f, context, obj, _) and - PointsToInternal::variablePointsTo(def.getInput(), prev, obj, _) and + exists(ObjectInternal defobj, Context prev, AttributeAssignment def, ObjectInternal useobj | + PointsToInternal::pointsTo(f, context, useobj, _) and + PointsToInternal::variablePointsTo(def.getInput(), prev, defobj, _) and PointsToInternal::pointsTo(def.getValue(), prev, value, origin) and name = def.getName() | - prev.getOuter*().getCall().getBasicBlock().reaches(context.getOuter*().getCall().getBasicBlock()) + prev.getOuter*().getCall().getBasicBlock().reaches(context.getOuter*().getCall().getBasicBlock()) and + useobj.(SelfInstanceInternal).getClass() = defobj.(SelfInstanceInternal).getClass() + or + def.getScope().getScope*().precedes(f.getScope().getScope*()) and + useobj.(SelfInstanceInternal).getClass() = defobj.(SelfInstanceInternal).getClass() or - def.getScope().getScope*().precedes(f.getScope().getScope*()) + def.getDefiningNode().getBasicBlock().dominates(f.getBasicBlock()) and defobj = useobj ) } @@ -1697,8 +1702,9 @@ module ModuleAttributes { private predicate importStarPointsTo(ImportStarRefinement def, string name, ObjectInternal value, CfgOrigin origin) { def.getVariable().getName() = "$" and exists(ImportStarNode imp, ModuleObjectInternal mod, CfgOrigin orig | + imp = def.getDefiningNode() and origin = orig.fix(imp) and - PointsToInternal::pointsTo(def.getDefiningNode().(ImportStarNode).getModule(), any(Context ctx | ctx.isImport()), mod, _) | + PointsToInternal::pointsTo(imp.getModule(), any(Context ctx | ctx.isImport()), mod, _) | /* Attribute from imported module */ InterModulePointsTo::module_exports_boolean(mod, name) = true and mod.attribute(name, value, orig) and diff --git a/python/ql/src/semmle/python/types/Builtins.qll b/python/ql/src/semmle/python/types/Builtins.qll index fc3108664ef8..e87e88c1c4b5 100644 --- a/python/ql/src/semmle/python/types/Builtins.qll +++ b/python/ql/src/semmle/python/types/Builtins.qll @@ -84,9 +84,6 @@ class Builtin extends @py_cobject { predicate isMethod() { this.getClass() = Builtin::special("MethodDescriptorType") or - this.getClass() = Builtin::special("BuiltinFunctionType") and - exists(Builtin cls | cls.isClass() and cls.getMember(_) = this) - or this.getClass().getName() = "wrapper_descriptor" } diff --git a/python/ql/src/semmle/python/types/ModuleObject.qll b/python/ql/src/semmle/python/types/ModuleObject.qll index 7519218703bd..a9d848839f6a 100644 --- a/python/ql/src/semmle/python/types/ModuleObject.qll +++ b/python/ql/src/semmle/python/types/ModuleObject.qll @@ -57,6 +57,15 @@ abstract class ModuleObject extends Object { ) } + predicate attributeRefersTo(string name, Object obj, ClassObject cls, ControlFlowNode origin) { + exists(Value val, CfgOrigin valorig | + theModule().(ModuleObjectInternal).attribute(name, val, valorig) and + obj = val.getSource() and + cls = val.getClass().getSource() and + origin = valorig.toCfgNode() + ) + } + /** Gets the package for this module. */ PackageObject getPackage() { this.getName().matches("%.%") and diff --git a/python/ql/src/semmle/python/types/Object.qll b/python/ql/src/semmle/python/types/Object.qll index 8d57865c492e..200bc16e001d 100644 --- a/python/ql/src/semmle/python/types/Object.qll +++ b/python/ql/src/semmle/python/types/Object.qll @@ -501,3 +501,20 @@ Object theUnknownType() { result.asBuiltin() = Builtin::unknownType() } +/* For backwards compatibility */ + +class SuperBoundMethod extends Object { + + string name; + + SuperBoundMethod() { + this.(AttrNode).getObject(name).pointsTo().getClass() = Value::named("super") + } + + override string toString() { + result = "super()." + name + } + +} + + diff --git a/python/ql/test/library-tests/objects/classes/Test.expected b/python/ql/test/library-tests/objects/classes/Test.expected deleted file mode 100644 index 9dae856c349e..000000000000 --- a/python/ql/test/library-tests/objects/classes/Test.expected +++ /dev/null @@ -1 +0,0 @@ -fail diff --git a/python/ql/test/library-tests/objects/classes/Test.ql b/python/ql/test/library-tests/objects/classes/Test.ql deleted file mode 100644 index eeb464d17f07..000000000000 --- a/python/ql/test/library-tests/objects/classes/Test.ql +++ /dev/null @@ -1,9 +0,0 @@ -import python - -private import semmle.python.objects.ObjectInternal -private import semmle.python.pointsto.PointsTo2 - -from ClassObjectInternal cls, ControlFlowNode f -where cls.introduced(f, _) -select cls.getName(), f - diff --git a/python/ql/test/library-tests/objects/classes/test.py b/python/ql/test/library-tests/objects/classes/test.py deleted file mode 100644 index b3ce272e850e..000000000000 --- a/python/ql/test/library-tests/objects/classes/test.py +++ /dev/null @@ -1,8 +0,0 @@ - - -class Foo(object): - pass - -class Bar(object): - pass - From f543adcd38215403cf482c0c41e850d8d50ddaaa Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 8 Apr 2019 15:46:44 +0100 Subject: [PATCH 033/128] Python points-to: Fix up matching arguments to parameters. --- .../src/semmle/python/objects/Callables.qll | 35 ++++++++++++++++++ .../src/semmle/python/objects/ObjectAPI.qll | 37 +++++++++++-------- .../semmle/python/security/TaintTracking.qll | 8 ++-- .../semmle/python/types/FunctionObject.qll | 4 +- 4 files changed, 63 insertions(+), 21 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index f870a0e7fd39..0208fc483c87 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -46,6 +46,10 @@ abstract class CallableObjectInternal extends ObjectInternal { CallNode getACall() { result = this.getACall(_) } + abstract NameNode getParameter(int n); + + abstract NameNode getParameterByName(string name); + } @@ -132,6 +136,14 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti ) } + override NameNode getParameter(int n) { + result.getNode() = this.getScope().getArg(n) + } + + override NameNode getParameterByName(string name) { + result.getNode() = this.getScope().getArgByName(name) + } + } @@ -229,6 +241,13 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc PointsTo::pointsTo(result.getFunction(), ctx, this, _) } + override NameNode getParameter(int n) { + none() + } + + override NameNode getParameterByName(string name) { + none() + } } @@ -289,6 +308,14 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod PointsTo::pointsTo(result.getFunction(), ctx, this, _) } + override NameNode getParameter(int n) { + none() + } + + override NameNode getParameterByName(string name) { + none() + } + } class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { @@ -352,6 +379,14 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { PointsTo::pointsTo(result.getFunction(), ctx, this, _) } + override NameNode getParameter(int n) { + result = this.getFunction().getParameter(n+1) + } + + override NameNode getParameterByName(string name) { + result = this.getFunction().getParameterByName(name) + } + } class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 3a3742329f05..5ef872da145c 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -57,21 +57,6 @@ class Value extends TObject { result = this.(ObjectInternal).getSource() } - /** Gets the `ControlFlowNode` that will be passed as the nth argument to `this` when called at `call`. - This predicate will correctly handle `x.y()`, treating `x` as the zeroth argument. - */ - ControlFlowNode getArgumentForCall(CallNode call, int n) { - // TO DO.. - none() - } - - /** Gets the `ControlFlowNode` that will be passed as the named argument to `this` when called at `call`. - This predicate will correctly handle `x.y()`, treating `x` as the self argument. - */ - ControlFlowNode getNamedArgumentForCall(CallNode call, string name) { - // TO DO.. - none() - } } class ModuleValue extends Value { @@ -130,6 +115,28 @@ class CallableValue extends Value { result = this.(PythonFunctionObjectInternal).getScope() } + NameNode getParameter(int n) { + result = this.(CallableObjectInternal).getParameter(n) + } + + NameNode getParameterByName(string name) { + result = this.(CallableObjectInternal).getParameterByName(name) + } + + ControlFlowNode getArgumentForCall(CallNode call, NameNode param) { + this.getACall() = call and + ( + exists(int n | call.getArg(n) = result and param = this.getParameter(n)) + or + exists(string name | call.getArgByName(name) = result and param = this.getParameterByName(name)) + ) + or + exists(BoundMethodObjectInternal bm | + result = getArgumentForCall(call, param) and + this = bm.getFunction() + ) + } + } class ClassValue extends Value { diff --git a/python/ql/src/semmle/python/security/TaintTracking.qll b/python/ql/src/semmle/python/security/TaintTracking.qll index 9d076b2558fd..fff65b988d5b 100755 --- a/python/ql/src/semmle/python/security/TaintTracking.qll +++ b/python/ql/src/semmle/python/security/TaintTracking.qll @@ -1204,13 +1204,13 @@ library module TaintFlowImplementation { exists(ParameterDefinition def | def.getDefiningNode() = param and exists(CallableValue func, CallNode call | - exists(int n | argument = func.getArgumentForCall(call, n) and param.getNode() = func.getScope().getArg(n)) + call.getFunction().pointsTo() = func and + callee = caller.getCallee(call) | + exists(int n | param = func.getParameter(n) and argument = call.getArg(n)) or - exists(string name | argument = func.getNamedArgumentForCall(call, name) and param.getNode() = func.getScope().getArgByName(name)) + exists(string name | param = func.getParameterByName(name) and argument = call.getArgByName(name)) or class_initializer_argument(_, _, call, func, argument, param) - | - callee = caller.getCallee(call) ) ) } diff --git a/python/ql/src/semmle/python/types/FunctionObject.qll b/python/ql/src/semmle/python/types/FunctionObject.qll index 4f210b1548d6..4087afc89420 100644 --- a/python/ql/src/semmle/python/types/FunctionObject.qll +++ b/python/ql/src/semmle/python/types/FunctionObject.qll @@ -71,14 +71,14 @@ abstract class FunctionObject extends Object { This predicate will correctly handle `x.y()`, treating `x` as the zeroth argument. */ ControlFlowNode getArgumentForCall(CallNode call, int n) { - result = theCallable().getArgumentForCall(call, n) + result = theCallable().getArgumentForCall(call, this.getFunction().getArg(n).asName().getAFlowNode()) } /** Gets the `ControlFlowNode` that will be passed as the named argument to `this` when called at `call`. This predicate will correctly handle `x.y()`, treating `x` as the self argument. */ ControlFlowNode getNamedArgumentForCall(CallNode call, string name) { - result = theCallable().getNamedArgumentForCall(call, name) + result = theCallable().getArgumentForCall(call, this.getFunction().getArgByName(name).asName().getAFlowNode()) } /** Whether this function never returns. This is an approximation. From b47c2dd0839148e48fdec9d24451cbb43762f31f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 8 Apr 2019 17:14:41 +0100 Subject: [PATCH 034/128] Python remove old MRO module and rename MRO2 to MRO. --- .../src/semmle/python/objects/Callables.qll | 2 +- .../ql/src/semmle/python/objects/Classes.qll | 2 +- .../src/semmle/python/objects/Constants.qll | 2 +- .../src/semmle/python/objects/Instances.qll | 2 +- .../ql/src/semmle/python/objects/Modules.qll | 2 +- .../semmle/python/objects/ObjectInternal.qll | 2 +- .../src/semmle/python/objects/Sequences.qll | 2 +- python/ql/src/semmle/python/pointsto/MRO.qll | 171 ++++--- python/ql/src/semmle/python/pointsto/MRO2.qll | 480 ------------------ .../src/semmle/python/pointsto/PointsTo.qll | 2 +- .../src/semmle/python/types/ClassObject.qll | 2 +- 11 files changed, 101 insertions(+), 568 deletions(-) delete mode 100644 python/ql/src/semmle/python/pointsto/MRO2.qll diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 0208fc483c87..48240f252ede 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -5,7 +5,7 @@ private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext -private import semmle.python.pointsto.MRO2 +private import semmle.python.pointsto.MRO private import semmle.python.types.Builtins diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 61a36618919f..6f0e63086453 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -5,7 +5,7 @@ private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext -private import semmle.python.pointsto.MRO2 +private import semmle.python.pointsto.MRO private import semmle.python.types.Builtins diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index cc15bc7ab852..5e1c2b6c8b67 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -3,7 +3,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo -private import semmle.python.pointsto.MRO2 +private import semmle.python.pointsto.MRO private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 0f7b6624b7a5..28d7128fba82 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -4,7 +4,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo -private import semmle.python.pointsto.MRO2 +private import semmle.python.pointsto.MRO private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 29d5ea5ecf5c..758cd267a775 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -3,7 +3,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo -private import semmle.python.pointsto.MRO2 +private import semmle.python.pointsto.MRO private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 8089cacb6556..55c499ac44a2 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -2,7 +2,7 @@ import python private import semmle.python.objects.TObject private import semmle.python.pointsto.PointsTo -private import semmle.python.pointsto.MRO2 +private import semmle.python.pointsto.MRO private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins import semmle.python.objects.Modules diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index 53ae9696dfde..7776eac9df06 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -8,7 +8,7 @@ private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext -private import semmle.python.pointsto.MRO2 +private import semmle.python.pointsto.MRO private import semmle.python.types.Builtins abstract class SequenceObjectInternal extends ObjectInternal { diff --git a/python/ql/src/semmle/python/pointsto/MRO.qll b/python/ql/src/semmle/python/pointsto/MRO.qll index 11ac81b120a7..af6bed2e02ac 100644 --- a/python/ql/src/semmle/python/pointsto/MRO.qll +++ b/python/ql/src/semmle/python/pointsto/MRO.qll @@ -17,23 +17,29 @@ */ import python -import semmle.python.pointsto.PointsTo + +private import semmle.python.objects.TObject +private import semmle.python.objects.ObjectInternal +private import semmle.python.pointsto.PointsTo +private import semmle.python.pointsto.PointsToContext +private import semmle.python.types.Builtins + cached private newtype TClassList = Empty() or - Cons(ClassObject head, TClassList tail) { + Cons(ClassObjectInternal head, TClassList tail) { required_cons(head, tail) } /* Keep ClassList finite and as small as possible */ -private predicate required_cons(ClassObject head, ClassList tail) { +private predicate required_cons(ClassObjectInternal head, ClassList tail) { tail = merge_of_linearization_of_bases(head) or - exists(ClassObject cls, int n | - head = cls.getBaseType(n) and tail = bases(cls, n+1) + exists(ClassObjectInternal cls, int n | + head = Types::getBase(cls, n) and tail = bases(cls, n+1) ) or - head = theObjectType() and tail = Empty() + head = ObjectInternal::builtin("object") and tail = Empty() or reverse_step(_, Cons(head, _), tail) or @@ -65,7 +71,7 @@ class ClassList extends TClassList { string contents() { this = Empty() and result = "" or - exists(ClassObject head | + exists(ClassObjectInternal head | head = this.getHead() | this.getTail() = Empty() and result = head.getName() or @@ -79,7 +85,7 @@ class ClassList extends TClassList { result = this.getTail().length() + 1 } - ClassObject getHead() { + ClassObjectInternal getHead() { this = Cons(result, _) } @@ -87,14 +93,18 @@ class ClassList extends TClassList { this = Cons(_, result) } - ClassObject getItem(int n) { + ClassObjectInternal getItem(int n) { n = 0 and result = this.getHead() or result = this.getTail().getItem(n-1) } + ClassObjectInternal getAnItem() { + result = this.getItem(_) + } + pragma [inline] - ClassList removeHead(ClassObject cls) { + ClassList removeHead(ClassObjectInternal cls) { this.getHead() = cls and result = this.getTail() or this.getHead() != cls and result = this @@ -102,29 +112,28 @@ class ClassList extends TClassList { this = Empty() and result = Empty() } - predicate legalMergeHead(ClassObject cls) { + predicate legalMergeHead(ClassObjectInternal cls) { this.getTail().doesNotContain(cls) or this = Empty() } - /** Use negative formulation for efficiency */ - predicate contains(ClassObject cls) { + predicate contains(ClassObjectInternal cls) { cls = this.getHead() or this.getTail().contains(cls) } /** Use negative formulation to avoid negative recursion */ - predicate doesNotContain(ClassObject cls) { - this.relevantForContains(cls) and - cls != this.getHead() and + predicate doesNotContain(ClassObjectInternal cls) { + this.relevantForContains(cls) and + cls != this.getHead() and this.getTail().doesNotContain(cls) or this = Empty() } - private predicate relevantForContains(ClassObject cls) { + private predicate relevantForContains(ClassObjectInternal cls) { exists(ClassListList list | list.getItem(_).getHead() = cls and list.getItem(_) = this @@ -136,36 +145,24 @@ class ClassList extends TClassList { ) } - ClassObject findDeclaringClass(string name) { - exists(ClassObject head | - head = this.getHead() and - not head = theUnknownType() | + ClassObjectInternal findDeclaringClass(string name) { + exists(ClassDecl head | + head = this.getHead().getClassDeclaration() | if head.declaresAttribute(name) then - result = head + result = this.getHead() else result = this.getTail().findDeclaringClass(name) ) } - Object lookup(string name) { - exists(ClassObject head | - head = this.getHead() and - not head = theUnknownType() | - if head.declaresAttribute(name) then - result = head.declaredAttribute(name) - else - result = this.getTail().lookup(name) - ) - } - predicate declares(string name) { - this.getHead().declaresAttribute(name) + this.getHead().getClassDeclaration().declaresAttribute(name) or this.getTail().declares(name) } - ClassList startingAt(ClassObject cls) { - exists(ClassObject head | + ClassList startingAt(ClassObjectInternal cls) { + exists(ClassObjectInternal head | head = this.getHead() | if head = cls then result = this @@ -180,12 +177,12 @@ class ClassList extends TClassList { /* Helpers for `deduplicate()` */ - int firstIndex(ClassObject cls) { + int firstIndex(ClassObjectInternal cls) { result = this.firstIndex(cls, 0) } /* Helper for firstIndex(cls), getting the first index of `cls` where result >= n */ - private int firstIndex(ClassObject cls, int n) { + private int firstIndex(ClassObjectInternal cls, int n) { this.getItem(n) = cls and result = n or this.getItem(n) != cls and result = this.firstIndex(cls, n+1) @@ -193,7 +190,7 @@ class ClassList extends TClassList { /** Holds if the class at `n` is a duplicate of an earlier position. */ private predicate duplicate(int n) { - exists(ClassObject cls | + exists(ClassObjectInternal cls | cls = this.getItem(n) and this.firstIndex(cls) < n ) } @@ -206,7 +203,7 @@ class ClassList extends TClassList { or this.duplicate(n) and result = this.deduplicate(n+1) or - exists(ClassObject cls | + exists(ClassObjectInternal cls | n = this.firstIndex(cls) and result = Cons(cls, this.deduplicate(n+1)) ) @@ -219,6 +216,19 @@ class ClassList extends TClassList { ClassList reverse() { reverse_step(this, Empty(), result) } + + boolean isSpecial() { + this = Empty() and result = false + or + exists(ClassDecl decl | + decl = this.getHead().getClassDeclaration() | + if decl.isSpecial() then + result = true + else + result = this.getTail().isSpecial() + ) + } + } private newtype TClassListList = @@ -233,13 +243,13 @@ private predicate required_list(ClassList head, ClassListList tail) { or head = bases(_) and tail = EmptyList() or - exists(ClassObject cls, int n | - head = new_style_mro(cls.getBaseType(n)) and + exists(ClassObjectInternal cls, int n | + head = Mro::newStyleMro(Types::getBase(cls, n)) and tail = list_of_linearization_of_bases_plus_bases(cls, n+1) ) or - exists(ClassObject cls, int n | - head = old_style_mro(cls.getBaseType(n)) and + exists(ClassObjectInternal cls, int n | + head = Mro::oldStyleMro(Types::getBase(cls, n)) and tail = list_old_style_base_mros(cls, n+1) ) } @@ -281,7 +291,7 @@ private class ClassListList extends TClassListList { result = this.getTail().getItem(n-1) } - private ClassObject getAHead() { + private ClassObjectInternal getAHead() { result = this.getHead().getHead() or result = this.getTail().getAHead() @@ -299,7 +309,7 @@ private class ClassListList extends TClassListList { /* Join ordering helper */ pragma [noinline] - predicate removedClassParts(ClassObject cls, ClassList removed_head, ClassListList removed_tail, int n) { + predicate removedClassParts(ClassObjectInternal cls, ClassList removed_head, ClassListList removed_tail, int n) { cls = this.bestMergeCandidate() and n = this.length()-1 and removed_head = this.getItem(n).removeHead(cls) and removed_tail = EmptyList() or @@ -310,7 +320,7 @@ private class ClassListList extends TClassListList { ) } - ClassListList remove(ClassObject cls) { + ClassListList remove(ClassObjectInternal cls) { exists(ClassList removed_head, ClassListList removed_tail | this.removedClassParts(cls, removed_head, removed_tail, 0) and result = ConsList(removed_head, removed_tail) @@ -319,24 +329,24 @@ private class ClassListList extends TClassListList { this = EmptyList() and result = EmptyList() } - predicate legalMergeCandidate(ClassObject cls, int n) { + predicate legalMergeCandidate(ClassObjectInternal cls, int n) { cls = this.getAHead() and n = this.length() or this.getItem(n).legalMergeHead(cls) and this.legalMergeCandidate(cls, n+1) } - predicate legalMergeCandidate(ClassObject cls) { + predicate legalMergeCandidate(ClassObjectInternal cls) { this.legalMergeCandidate(cls, 0) } - predicate illegalMergeCandidate(ClassObject cls) { + predicate illegalMergeCandidate(ClassObjectInternal cls) { cls = this.getAHead() and this.getItem(_).getTail().contains(cls) } - ClassObject bestMergeCandidate(int n) { - exists(ClassObject head | + ClassObjectInternal bestMergeCandidate(int n) { + exists(ClassObjectInternal head | head = this.getItem(n).getHead() | legalMergeCandidate(head) and result = head @@ -345,7 +355,7 @@ private class ClassListList extends TClassListList { ) } - ClassObject bestMergeCandidate() { + ClassObjectInternal bestMergeCandidate() { result = this.bestMergeCandidate(0) } @@ -381,53 +391,42 @@ private predicate need_flattening(ClassListList list) { ) } -private ClassList bases(ClassObject cls) { +private ClassList bases(ClassObjectInternal cls) { result = bases(cls, 0) } -private ClassList bases(ClassObject cls, int n) { - result = Cons(cls.getBaseType(n), bases(cls, n+1)) +private ClassList bases(ClassObjectInternal cls, int n) { + result = Cons(Types::getBase(cls, n), bases(cls, n+1)) or - result = Empty() and n = PointsTo::Types::class_base_count(cls) + result = Empty() and n = Types::base_count(cls) } -private ClassListList list_of_linearization_of_bases_plus_bases(ClassObject cls) { +private ClassListList list_of_linearization_of_bases_plus_bases(ClassObjectInternal cls) { result = list_of_linearization_of_bases_plus_bases(cls, 0) } -private ClassListList list_of_linearization_of_bases_plus_bases(ClassObject cls, int n) { - result = ConsList(bases(cls), EmptyList()) and n = PointsTo::Types::class_base_count(cls) +private ClassListList list_of_linearization_of_bases_plus_bases(ClassObjectInternal cls, int n) { + result = ConsList(bases(cls), EmptyList()) and n = Types::base_count(cls) or exists(ClassListList partial | partial = list_of_linearization_of_bases_plus_bases(cls, n+1) and - result = ConsList(new_style_mro(cls.getBaseType(n)), partial) + result = ConsList(Mro::newStyleMro(Types::getBase(cls, n)), partial) ) } -private ClassList merge_of_linearization_of_bases(ClassObject cls) { +private ClassList merge_of_linearization_of_bases(ClassObjectInternal cls) { result = list_of_linearization_of_bases_plus_bases(cls).merge() } -cached ClassList new_style_mro(ClassObject cls) { - cls = theObjectType() and result = Cons(cls, Empty()) - or - result = Cons(cls, merge_of_linearization_of_bases(cls)) -} - -cached ClassList old_style_mro(ClassObject cls) { - PointsTo::Types::is_new_style_bool(cls) = false and - result = Cons(cls, list_old_style_base_mros(cls).flatten()).(ClassList).deduplicate() -} - -private ClassListList list_old_style_base_mros(ClassObject cls) { +private ClassListList list_old_style_base_mros(ClassObjectInternal cls) { result = list_old_style_base_mros(cls, 0) } pragma [nomagic] -private ClassListList list_old_style_base_mros(ClassObject cls, int n) { - n = PointsTo::Types::class_base_count(cls) and result = EmptyList() +private ClassListList list_old_style_base_mros(ClassObjectInternal cls, int n) { + n = Types::base_count(cls) and result = EmptyList() or - result = ConsList(old_style_mro(cls.getBaseType(n)), list_old_style_base_mros(cls, n+1)) + result = ConsList(Mro::oldStyleMro(Types::getBase(cls, n)), list_old_style_base_mros(cls, n+1)) } /** Holds if the pair `reversed_mro`, `remaining_list` represents a step in the C3 merge operation @@ -437,7 +436,7 @@ private predicate merge_step(ClassList reversed_mro, ClassListList remaining_lis remaining_list = list_of_linearization_of_bases_plus_bases(_) and reversed_mro = Empty() and remaining_list = original or /* Removes the best merge candidate from `remaining_list` and prepends it to `reversed_mro` */ - exists(ClassObject head, ClassList prev_reverse_mro, ClassListList prev_list | + exists(ClassObjectInternal head, ClassList prev_reverse_mro, ClassListList prev_list | merge_step(prev_reverse_mro, prev_list, original) and head = prev_list.bestMergeCandidate() and reversed_mro = Cons(head, prev_reverse_mro) and @@ -458,9 +457,23 @@ private predicate needs_reversing(ClassList lst) { private predicate reverse_step(ClassList lst, ClassList remainder, ClassList reversed) { needs_reversing(lst) and remainder = lst and reversed = Empty() or - exists(ClassObject head, ClassList tail | + exists(ClassObjectInternal head, ClassList tail | reversed = Cons(head, tail) and reverse_step(lst, Cons(head, remainder), tail) ) } +module Mro { + + cached ClassList newStyleMro(ClassObjectInternal cls) { + cls = ObjectInternal::builtin("object") and result = Cons(cls, Empty()) + or + result = Cons(cls, merge_of_linearization_of_bases(cls)) + } + + cached ClassList oldStyleMro(ClassObjectInternal cls) { + Types::isOldStyle(cls) and + result = Cons(cls, list_old_style_base_mros(cls).flatten()).(ClassList).deduplicate() + } + +} \ No newline at end of file diff --git a/python/ql/src/semmle/python/pointsto/MRO2.qll b/python/ql/src/semmle/python/pointsto/MRO2.qll deleted file mode 100644 index 4c336c1abdbe..000000000000 --- a/python/ql/src/semmle/python/pointsto/MRO2.qll +++ /dev/null @@ -1,480 +0,0 @@ -/** Classes and predicates for computing the Method Resolution Order (MRO) of classes. - * Supports both old-style (diamond) inheritance and new-style (C3 linearization) inheritance. - */ - -/* - * Implementation of the C3 linearization algorithm. - * See https://en.wikipedia.org/wiki/C3_linearization - * - * The key operation is merge, which takes a list of lists and produces a list. - * We implement it as the method `ClassListList.merge()` - * - * To support that we need to determine the best candidate to extract from a list of lists, - * implemented as `ClassListList.bestMergeCandidate()` - * - * The following code is designed to implement those operations - * without negation and as efficiently as possible. - */ - -import python - -private import semmle.python.objects.TObject -private import semmle.python.objects.ObjectInternal -private import semmle.python.pointsto.PointsTo -private import semmle.python.pointsto.PointsToContext -private import semmle.python.types.Builtins - - -cached private newtype TClassList = Empty() - or - Cons(ClassObjectInternal head, TClassList tail) { - required_cons(head, tail) - } - -/* Keep ClassList finite and as small as possible */ -private predicate required_cons(ClassObjectInternal head, ClassList tail) { - tail = merge_of_linearization_of_bases(head) - or - exists(ClassObjectInternal cls, int n | - head = Types::getBase(cls, n) and tail = bases(cls, n+1) - ) - or - head = ObjectInternal::builtin("object") and tail = Empty() - or - reverse_step(_, Cons(head, _), tail) - or - exists(ClassListList list | - merge_step(tail, list, _) and - head = list.bestMergeCandidate() - ) - or - exists(ClassList list, int n | - n = list.firstIndex(head) and - tail = list.deduplicate(n+1) - ) - or - exists(ClassListList list, int n | - head = list.getHead().getItem(n) and - tail = flatten_list(list, n+1) - ) - or - tail = list_old_style_base_mros(head).flatten() -} - -/** A list of classes, used to represent the MRO of a class */ -class ClassList extends TClassList { - - string toString() { - result = "[" + this.contents() + "]" - } - - string contents() { - this = Empty() and result = "" - or - exists(ClassObjectInternal head | - head = this.getHead() | - this.getTail() = Empty() and result = head.getName() - or - this.getTail() != Empty() and result = head.getName() + ", " + this.getTail().contents() - ) - } - - int length() { - this = Empty() and result = 0 - or - result = this.getTail().length() + 1 - } - - ClassObjectInternal getHead() { - this = Cons(result, _) - } - - ClassList getTail() { - this = Cons(_, result) - } - - ClassObjectInternal getItem(int n) { - n = 0 and result = this.getHead() - or - result = this.getTail().getItem(n-1) - } - - ClassObjectInternal getAnItem() { - result = this.getItem(_) - } - - pragma [inline] - ClassList removeHead(ClassObjectInternal cls) { - this.getHead() = cls and result = this.getTail() - or - this.getHead() != cls and result = this - or - this = Empty() and result = Empty() - } - - predicate legalMergeHead(ClassObjectInternal cls) { - this.getTail().doesNotContain(cls) - or - this = Empty() - } - - /** Use negative formulation for efficiency */ - predicate contains(ClassObjectInternal cls) { - cls = this.getHead() - or - this.getTail().contains(cls) - } - - /** Use negative formulation to avoid negative recursion */ - predicate doesNotContain(ClassObjectInternal cls) { - this.relevantForContains(cls) and - cls != this.getHead() and - this.getTail().doesNotContain(cls) - or - this = Empty() - } - - private predicate relevantForContains(ClassObjectInternal cls) { - exists(ClassListList list | - list.getItem(_).getHead() = cls and - list.getItem(_) = this - ) - or - exists(ClassList l | - l.relevantForContains(cls) and - this = l.getTail() - ) - } - - ClassObjectInternal findDeclaringClass(string name) { - exists(ClassDecl head | - head = this.getHead().getClassDeclaration() | - if head.declaresAttribute(name) then - result = this.getHead() - else - result = this.getTail().findDeclaringClass(name) - ) - } - - predicate declares(string name) { - this.getHead().getClassDeclaration().declaresAttribute(name) - or - this.getTail().declares(name) - } - - ClassList startingAt(ClassObjectInternal cls) { - exists(ClassObjectInternal head | - head = this.getHead() | - if head = cls then - result = this - else - result = this.getTail().startingAt(cls) - ) - } - - ClassList deduplicate() { - result = this.deduplicate(0) - } - - /* Helpers for `deduplicate()` */ - - int firstIndex(ClassObjectInternal cls) { - result = this.firstIndex(cls, 0) - } - - /* Helper for firstIndex(cls), getting the first index of `cls` where result >= n */ - private int firstIndex(ClassObjectInternal cls, int n) { - this.getItem(n) = cls and result = n - or - this.getItem(n) != cls and result = this.firstIndex(cls, n+1) - } - - /** Holds if the class at `n` is a duplicate of an earlier position. */ - private predicate duplicate(int n) { - exists(ClassObjectInternal cls | - cls = this.getItem(n) and this.firstIndex(cls) < n - ) - } - - /** Gets a class list which is the de-duplicated form of the list containing elements of - * this list from `n` onwards. - */ - ClassList deduplicate(int n) { - n = this.length() and result = Empty() - or - this.duplicate(n) and result = this.deduplicate(n+1) - or - exists(ClassObjectInternal cls | - n = this.firstIndex(cls) and - result = Cons(cls, this.deduplicate(n+1)) - ) - } - - predicate isEmpty() { - this = Empty() - } - - ClassList reverse() { - reverse_step(this, Empty(), result) - } - - boolean isSpecial() { - this = Empty() and result = false - or - exists(ClassDecl decl | - decl = this.getHead().getClassDeclaration() | - if decl.isSpecial() then - result = true - else - result = this.getTail().isSpecial() - ) - } - -} - -private newtype TClassListList = - EmptyList() or - ConsList(TClassList head, TClassListList tail) { - required_list(head, tail) - } - -/* Keep ClassListList finite and as small as possible */ -private predicate required_list(ClassList head, ClassListList tail) { - any(ClassListList x).removedClassParts(_, head, tail, _) - or - head = bases(_) and tail = EmptyList() - or - exists(ClassObjectInternal cls, int n | - head = Mro::newStyleMro(Types::getBase(cls, n)) and - tail = list_of_linearization_of_bases_plus_bases(cls, n+1) - ) - or - exists(ClassObjectInternal cls, int n | - head = Mro::oldStyleMro(Types::getBase(cls, n)) and - tail = list_old_style_base_mros(cls, n+1) - ) -} - -private class ClassListList extends TClassListList { - - string toString() { - result = "[" + this.contents() + "]" - } - - string contents() { - this = EmptyList() and result = "" - or - exists(ClassList head | - head = this.getHead() | - this.getTail() = EmptyList() and result = head.toString() - or - this.getTail() != EmptyList() and result = head.toString() + ", " + this.getTail().contents() - ) - } - - int length() { - this = EmptyList() and result = 0 - or - result = this.getTail().length() + 1 - } - - ClassList getHead() { - this = ConsList(result, _) - } - - ClassListList getTail() { - this = ConsList(_, result) - } - - ClassList getItem(int n) { - n = 0 and result = this.getHead() - or - result = this.getTail().getItem(n-1) - } - - private ClassObjectInternal getAHead() { - result = this.getHead().getHead() - or - result = this.getTail().getAHead() - } - - pragma [nomagic] - ClassList merge() { - exists(ClassList reversed | - merge_step(reversed, EmptyList(), this) and - result = reversed.reverse() - ) - or - this = EmptyList() and result = Empty() - } - - /* Join ordering helper */ - pragma [noinline] - predicate removedClassParts(ClassObjectInternal cls, ClassList removed_head, ClassListList removed_tail, int n) { - cls = this.bestMergeCandidate() and n = this.length()-1 and - removed_head = this.getItem(n).removeHead(cls) and removed_tail = EmptyList() - or - exists(ClassList prev_head, ClassListList prev_tail | - this.removedClassParts(cls, prev_head, prev_tail, n+1) and - removed_head = this.getItem(n).removeHead(cls) and - removed_tail = ConsList(prev_head, prev_tail) - ) - } - - ClassListList remove(ClassObjectInternal cls) { - exists(ClassList removed_head, ClassListList removed_tail | - this.removedClassParts(cls, removed_head, removed_tail, 0) and - result = ConsList(removed_head, removed_tail) - ) - or - this = EmptyList() and result = EmptyList() - } - - predicate legalMergeCandidate(ClassObjectInternal cls, int n) { - cls = this.getAHead() and n = this.length() - or - this.getItem(n).legalMergeHead(cls) and - this.legalMergeCandidate(cls, n+1) - } - - predicate legalMergeCandidate(ClassObjectInternal cls) { - this.legalMergeCandidate(cls, 0) - } - - predicate illegalMergeCandidate(ClassObjectInternal cls) { - cls = this.getAHead() and - this.getItem(_).getTail().contains(cls) - } - - ClassObjectInternal bestMergeCandidate(int n) { - exists(ClassObjectInternal head | - head = this.getItem(n).getHead() - | - legalMergeCandidate(head) and result = head - or - illegalMergeCandidate(head) and result = this.bestMergeCandidate(n+1) - ) - } - - ClassObjectInternal bestMergeCandidate() { - result = this.bestMergeCandidate(0) - } - - /** Gets a ClassList representing the this list of list flattened into a single list. - * Used for old-style MRO computation. - */ - ClassList flatten() { - this = EmptyList() and result = Empty() - or - result = flatten_list(this, 0) - } - -} - -private ClassList flatten_list(ClassListList list, int n) { - need_flattening(list) and - exists(ClassList head, ClassListList tail | - list = ConsList(head, tail) - | - n = head.length() and result = tail.flatten() - or - result = Cons(head.getItem(n), flatten_list(list, n+1)) - ) -} - -/* Restrict flattening to those lists that need to be flattened */ -private predicate need_flattening(ClassListList list) { - list = list_old_style_base_mros(_) - or - exists(ClassListList toflatten | - need_flattening(toflatten) and - list = toflatten.getTail() - ) -} - -private ClassList bases(ClassObjectInternal cls) { - result = bases(cls, 0) -} - -private ClassList bases(ClassObjectInternal cls, int n) { - result = Cons(Types::getBase(cls, n), bases(cls, n+1)) - or - result = Empty() and n = Types::base_count(cls) -} - -private ClassListList list_of_linearization_of_bases_plus_bases(ClassObjectInternal cls) { - result = list_of_linearization_of_bases_plus_bases(cls, 0) -} - -private ClassListList list_of_linearization_of_bases_plus_bases(ClassObjectInternal cls, int n) { - result = ConsList(bases(cls), EmptyList()) and n = Types::base_count(cls) - or - exists(ClassListList partial | - partial = list_of_linearization_of_bases_plus_bases(cls, n+1) and - result = ConsList(Mro::newStyleMro(Types::getBase(cls, n)), partial) - ) -} - -private ClassList merge_of_linearization_of_bases(ClassObjectInternal cls) { - result = list_of_linearization_of_bases_plus_bases(cls).merge() -} - -private ClassListList list_old_style_base_mros(ClassObjectInternal cls) { - result = list_old_style_base_mros(cls, 0) -} - -pragma [nomagic] -private ClassListList list_old_style_base_mros(ClassObjectInternal cls, int n) { - n = Types::base_count(cls) and result = EmptyList() - or - result = ConsList(Mro::oldStyleMro(Types::getBase(cls, n)), list_old_style_base_mros(cls, n+1)) -} - -/** Holds if the pair `reversed_mro`, `remaining_list` represents a step in the C3 merge operation - * of computing the C3 linearization of `original`. - */ -private predicate merge_step(ClassList reversed_mro, ClassListList remaining_list, ClassListList original) { - remaining_list = list_of_linearization_of_bases_plus_bases(_) and reversed_mro = Empty() and remaining_list = original - or - /* Removes the best merge candidate from `remaining_list` and prepends it to `reversed_mro` */ - exists(ClassObjectInternal head, ClassList prev_reverse_mro, ClassListList prev_list | - merge_step(prev_reverse_mro, prev_list, original) and - head = prev_list.bestMergeCandidate() and - reversed_mro = Cons(head, prev_reverse_mro) and - remaining_list = prev_list.remove(head) - ) - or - merge_step(reversed_mro, ConsList(Empty(), remaining_list), original) -} - -/* Helpers for `ClassList.reverse()` */ - -private predicate needs_reversing(ClassList lst) { - merge_step(lst, EmptyList(), _) - or - lst = Empty() -} - -private predicate reverse_step(ClassList lst, ClassList remainder, ClassList reversed) { - needs_reversing(lst) and remainder = lst and reversed = Empty() - or - exists(ClassObjectInternal head, ClassList tail | - reversed = Cons(head, tail) and - reverse_step(lst, Cons(head, remainder), tail) - ) -} - -module Mro { - - cached ClassList newStyleMro(ClassObjectInternal cls) { - cls = ObjectInternal::builtin("object") and result = Cons(cls, Empty()) - or - result = Cons(cls, merge_of_linearization_of_bases(cls)) - } - - cached ClassList oldStyleMro(ClassObjectInternal cls) { - Types::isOldStyle(cls) and - result = Cons(cls, list_old_style_base_mros(cls).flatten()).(ClassList).deduplicate() - } - -} \ No newline at end of file diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 74769a4ce6c8..f61986d541b8 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -4,7 +4,7 @@ private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.Filters private import semmle.python.pointsto.PointsToContext -private import semmle.python.pointsto.MRO2 +private import semmle.python.pointsto.MRO private import semmle.python.types.Builtins /* Use this version for speed */ diff --git a/python/ql/src/semmle/python/types/ClassObject.qll b/python/ql/src/semmle/python/types/ClassObject.qll index bd34f5f39949..6de3ca3a27d5 100644 --- a/python/ql/src/semmle/python/types/ClassObject.qll +++ b/python/ql/src/semmle/python/types/ClassObject.qll @@ -2,7 +2,7 @@ import python private import semmle.python.objects.Classes private import semmle.python.objects.Instances private import semmle.python.pointsto.PointsTo -private import semmle.python.pointsto.MRO2 +private import semmle.python.pointsto.MRO private import semmle.python.types.Builtins From 5644072a8ded19cb77f94fa26ee4bbde54348f1e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 8 Apr 2019 18:05:05 +0100 Subject: [PATCH 035/128] Python points-to: make sure builtin tuples are visible. --- python/ql/src/semmle/python/objects/ObjectInternal.qll | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 55c499ac44a2..d54fa6af3c0f 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -365,6 +365,8 @@ module ObjectInternal { result = TBuiltinModuleObject(b) or result = TBuiltinMethodObject(b) + or + result = TBuiltinTuple(b) } ObjectInternal classMethod() { From 0b2421eb23fbf2ecb7f7e9e0566c28836d12bdfb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 8 Apr 2019 18:05:26 +0100 Subject: [PATCH 036/128] Python points-to. Tidy up a bit. --- python/ql/src/semmle/python/pointsto/PointsTo.qll | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index f61986d541b8..9574dcb7b38a 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -129,7 +129,7 @@ module PointsTo { pointsToValue(f, context, value, origin) and cls = value.getClass().getSource() | obj = value.getSource() or - not exists(value.getSource()) and obj = origin and not cls = theBoundMethodType() + not exists(value.getSource()) and obj = origin ) or /* Backwards compatibility for *args and **kwargs */ @@ -399,7 +399,6 @@ cached module PointsToInternal { InterProceduralPointsTo::parameter_points_to(def, context, value, origin) or assignment_points_to(def, context, value, origin) - //// TO DO... or self_parameter_points_to(def, context, value, origin) or @@ -606,13 +605,6 @@ module InterModulePointsTo { (mod.getSourceModule() != f.getEnclosingModule() or mod.isBuiltin()) and mod.attribute(name, value, orig) and origin = orig.asCfgNodeOrHere(f) - // TO DO... $ variables. - //mod.getSourceModule() = f.getEnclosingModule() and - //not exists(EssaVariable var | var.getSourceVariable().getName() = name and var.getAUse() = f) and - //exists(EssaVariable dollar | - // isModuleStateVariable(dollar) and dollar.getAUse() = f and - // SSA::ssa_variable_named_attribute_points_to(dollar, context, name, value, orig) - //) ) } From 55eac7d555048e07305ae3066790120aa14915cc Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 9 Apr 2019 15:50:50 +0100 Subject: [PATCH 037/128] Python points-to: Fix up handling of recursive 'from ... import *'. --- .../src/semmle/python/objects/Constants.qll | 6 +- .../ql/src/semmle/python/objects/Modules.qll | 9 -- .../src/semmle/python/objects/ObjectAPI.qll | 4 - .../ql/src/semmle/python/objects/TObject.qll | 21 ++- .../src/semmle/python/pointsto/PointsTo.qll | 142 +++++++++++------- 5 files changed, 110 insertions(+), 72 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index 5e1c2b6c8b67..62ee913826e2 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -191,7 +191,11 @@ class IntObjectInternal extends ConstantObjectInternal, TInt { class FloatObjectInternal extends ConstantObjectInternal, TFloat { override string toString() { - result = "float " + this.floatValue().toString() + if this.floatValue() = this.floatValue().floor() then ( + result = "float " + this.floatValue().toString() + ".0" + ) else ( + result = "float " + this.floatValue().toString() + ) } override predicate introduced(ControlFlowNode node, PointsToContext context) { diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 758cd267a775..e999b562faf3 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -189,15 +189,6 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { } -/** Get the ESSA pseudo-variable used to retain module state - * during module initialization. Module attributes are handled - * as attributes of this variable, allowing the SSA form to track - * mutations of the module during its creation. - */ -private predicate isModuleStateVariable(EssaVariable var) { - var.getName() = "$" and var.getScope() instanceof Module -} - class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { override Builtin getBuiltin() { diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 5ef872da145c..abc32b187e19 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -22,10 +22,6 @@ class Value extends TObject { PointsToInternal::pointsTo(result, _, this, _) } - predicate pointsTo(ControlFlowNode node, PointsToContext context, ControlFlowNode origin) { - PointsToInternal::pointsTo(node, context, this, origin) - } - Value getClass() { result = this.(ObjectInternal).getClass() } diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 033db340449c..f937eb03e700 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -84,7 +84,7 @@ newtype TObject = or TSpecificInstance(ControlFlowNode instantiation, ClassObjectInternal cls, PointsToContext context) { PointsToInternal::pointsTo(instantiation.(CallNode).getFunction(), context, cls, _) and - cls.isSpecial() = false + cls.isSpecial() = false and cls.getClassDeclaration().callReturnsInstance() or literal_instantiation(instantiation, cls, context) } @@ -275,8 +275,6 @@ private predicate neither_class_nor_static_method(Function f) { ) } - - library class ClassDecl extends @py_object { ClassDecl() { @@ -330,5 +328,22 @@ library class ClassDecl extends @py_object { this = Builtin::builtin("float") } + predicate callReturnsInstance() { + exists(Class pycls | + pycls = this.getClass() | + /* Django does this, so we need to account for it */ + not exists(Function init, LocalVariable self | + /* `self.__class__ = ...` in the `__init__` method */ + pycls.getInitMethod() = init and + self.isSelf() and self.getScope() = init and + exists(AttrNode a | a.isStore() and a.getObject("__class__") = self.getAUse()) + ) + and + not exists(Function new | new.getName() = "__new__" and new.getScope() = pycls) + ) + or + this instanceof Builtin + } + } diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 9574dcb7b38a..2f701649df27 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -661,7 +661,7 @@ module InterModulePointsTo { imp = def.getDefiningNode() and PointsToInternal::pointsTo(imp.getModule(), context, mod, _) and name = def.getSourceVariable().getName() and - module_exports_boolean(mod, name) = true and + moduleExportsBoolean(mod, name) = true and mod.attribute(name, value, orig) and origin = orig.fix(imp) ) @@ -679,7 +679,7 @@ module InterModulePointsTo { var = def.getInput() and exists(ModuleObjectInternal mod | PointsToInternal::pointsTo(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) | - module_exports_boolean(mod, var.getSourceVariable().getName()) = false + moduleExportsBoolean(mod, var.getSourceVariable().getName()) = false or exists(Module m, string name | m = mod.getSourceModule() and name = var.getSourceVariable().getName() | @@ -698,52 +698,77 @@ module InterModulePointsTo { ) } - private predicate ofInterestInModule(ModuleObjectInternal mod, string name) { - exists(ImportStarNode isn, Module m | - m = mod.getSourceModule() and - isn.getScope() = m and - exists(EssaVariable var | var.getAUse() = isn and var.getName() = name) + predicate ofInterestInExports(ModuleObjectInternal mod, string name) { + exists(ImportStarNode imp, ImportStarRefinement def | + imp = def.getDefiningNode() and + PointsToInternal::pointsTo(imp.getModule(), any(Context ctx | ctx.isImport()), mod, _) | + def.getVariable().getName() = "$" and + ModuleAttributes::attributePointsTo(def.getInput().getDefinition(), name, _, _) + or + def.getVariable().getName() = name and not name = "$" and not name = "*" ) - } - - private predicate ofInterestInExports(ModuleObjectInternal mod, string name) { - exists(ModuleObjectInternal importer | - importsByImportStar(importer, mod) and - ofInterestInModule(importer, name) + or + exists(PackageObjectInternal package | + ofInterestInExports(package, name) and + package.getInitModule() = mod ) } - boolean module_exports_boolean(ModuleObjectInternal mod, string name) { - ofInterestInExports(mod, name) and + private boolean pythonModuleExportsBoolean(PythonModuleObjectInternal mod, string name) { exists(Module src | src = mod.getSourceModule() | - if exists(SsaVariable var | name = var.getId() and var.getAUse() = src.getANormalExit()) then - result = true - else ( - exists(ImportStarNode isn, ModuleObjectInternal imported | - isn.getScope() = src and - PointsToInternal::pointsTo(isn.getModule(), _, imported, _) and - result = module_exports_boolean(imported, name) - ) + src.declaredInAll(name) and result = true + or + src.declaredInAll(_) and not src.declaredInAll(name) and + ofInterestInExports(mod, name) and result = false + or + not src.declaredInAll(_) and + exists(ObjectInternal val | + ModuleAttributes::pointsToAtExit(src, name, val, _) | + val = ObjectInternal::undefined() and result = false or - not exists(ImportStarNode isn |isn.getScope() = src) and result = false + val != ObjectInternal::undefined() and result = true ) ) - or - ofInterestInExports(mod, name) and + } + + private boolean packageExportsBoolean(PackageObjectInternal mod, string name) { exists(Folder folder | - mod.(PackageObjectInternal).hasNoInitModule() and - folder = mod.(PackageObjectInternal).getFolder() | - if (exists(folder.getChildContainer(name)) or exists(folder.getFile(name + ".py"))) then - result = true - else - result = false + folder = mod.getFolder() | + exportsSubmodule(folder, name) and result = true + or + not exportsSubmodule(folder, name) and result = moduleExportsBoolean(mod.getInitModule(), name) + or + mod.hasNoInitModule() and not exportsSubmodule(folder, name) and + ofInterestInExports(mod, name) and result = false + ) + } + + private predicate exportsSubmodule(Folder folder, string name) { + name.regexpMatch("\\p{L}(\\p{L}|\\d|_)*") and + ( + exists(Folder child | child = folder.getChildContainer(name)) + or + exists(folder.getFile(name + ".py")) ) + } + + boolean builtinModuleExportsBoolean(BuiltinModuleObjectInternal mod, string name) { + exists(Builtin bltn | + bltn = mod.getBuiltin() | + exists(bltn.getMember(name)) and result = true + or + ofInterestInExports(mod, name) and not exists(bltn.getMember(name)) and result = false + ) + } + + boolean moduleExportsBoolean(ModuleObjectInternal mod, string name) { + result = pythonModuleExportsBoolean(mod, name) or - name = "__name__" and result = true + result = packageExportsBoolean(mod, name) or - exists(mod.(BuiltinModuleObjectInternal).getBuiltin().getMember(name)) and result = true + result = builtinModuleExportsBoolean(mod, name) } } @@ -1678,11 +1703,11 @@ module ModuleAttributes { if exists(varAtExit(mod, name)) then ( PointsToInternal::variablePointsTo(varAtExit(mod, name), any(Context c | c.isImport()), value, origin) ) else ( - moduleAttributePointsTo(moduleStateVarAtExit(mod), name, value, origin) + attributePointsTo(moduleStateVarAtExit(mod), name, value, origin) ) } - private predicate moduleAttributePointsTo(EssaVariable var, string name, ObjectInternal value, CfgOrigin origin) { + predicate attributePointsTo(EssaVariable var, string name, ObjectInternal value, CfgOrigin origin) { importStarPointsTo(var.getDefinition(), name, value, origin) or callsitePointsTo(var.getDefinition(), name, value, origin) @@ -1693,18 +1718,20 @@ module ModuleAttributes { pragma [noinline] private predicate importStarPointsTo(ImportStarRefinement def, string name, ObjectInternal value, CfgOrigin origin) { def.getVariable().getName() = "$" and - exists(ImportStarNode imp, ModuleObjectInternal mod, CfgOrigin orig | + exists(ImportStarNode imp, ModuleObjectInternal mod | imp = def.getDefiningNode() and - origin = orig.fix(imp) and - PointsToInternal::pointsTo(imp.getModule(), any(Context ctx | ctx.isImport()), mod, _) | + PointsToInternal::pointsTo(imp.getModule(), any(Context ctx | ctx.isImport()), mod, _) + | /* Attribute from imported module */ - InterModulePointsTo::module_exports_boolean(mod, name) = true and - mod.attribute(name, value, orig) and - not exists(Variable v | v.getId() = name and v.getScope() = imp.getScope()) + exists(CfgOrigin orig | + InterModulePointsTo::moduleExportsBoolean(mod, name) = true and + mod.attribute(name, value, orig) and origin = orig.fix(imp) and + not exists(Variable v | v.getId() = name and v.getScope() = imp.getScope()) + ) or /* Retain value held before import */ - InterModulePointsTo::module_exports_boolean(mod, name) = false and - moduleAttributePointsTo(def.getInput().getDefinition(), name, value, origin) + InterModulePointsTo::moduleExportsBoolean(mod, name) = false and + attributePointsTo(def.getInput(), name, value, origin) ) } @@ -1718,25 +1745,30 @@ module ModuleAttributes { exists(EssaVariable var, Function func, PointsToContext callee | InterProceduralPointsTo::callsite_calls_function(def.getCall(), _, func, callee, _) and var = moduleStateVariable(func.getANormalExit()) and - moduleAttributePointsTo(var, name, value, origin) + attributePointsTo(var, name, value, origin) ) } + /** Holds if the attribute name of the implicit '$' variable refers to `value` at the start of the scope. + * Since it cannot refer to any actual value, it is set to "undefined" for sub module names. + */ pragma [noinline] predicate scopeEntryPointsTo(ScopeEntryDefinition def, string name, ObjectInternal value, CfgOrigin origin) { def.getVariable().getName() = "$" and - /* Transfer from another scope */ - exists(EssaVariable var, PointsToContext outer | - InterProceduralPointsTo::scope_entry_value_transfer(var, outer, def, _) and - moduleAttributePointsTo(var, name, value, origin) - ) - or - def.getVariable().getName() = "$" and - exists(PackageObjectInternal package | - package.getSourceModule() = def.getScope() and - exists(package.submodule(name)) and + exists(Module m | + def.getScope() = m and + not exists(EssaVariable named | named.getName() = name and named.getScope() = m) and + not name = "$" and not name = "*" and value = ObjectInternal::undefined() and origin = CfgOrigin::unknown() + | + m.isPackageInit() and exists(m.getPackage().getSubModule(name)) + or + not m.declaredInAll(_) and + exists(PythonModuleObjectInternal mod | + mod.getSourceModule() = m and + InterModulePointsTo::ofInterestInExports(mod, name) + ) ) } From b8fb3e3e61f2c6824b015aa11b27b4249454f807 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 9 Apr 2019 16:20:01 +0100 Subject: [PATCH 038/128] Python points-to: Distinguish between class attribute access and lookup. Fixes handling of classmethods. --- .../ql/src/semmle/python/objects/Classes.qll | 24 +++++++++++++++---- .../src/semmle/python/objects/Instances.qll | 4 ++-- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 6f0e63086453..ef4d01a8c445 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -40,10 +40,24 @@ abstract class ClassObjectInternal extends ObjectInternal { override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { instance = this and PointsToInternal::attributeRequired(this, name) and - this.attribute(name, descriptor, _) and + this.lookup(name, descriptor, _) and descriptor.isDescriptor() = true } + abstract predicate lookup(string name, ObjectInternal value, CfgOrigin origin); + + /** Approximation to descriptor protocol, skipping meta-descriptor protocol */ + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + exists(ObjectInternal descriptor, CfgOrigin desc_origin | + this.lookup(name, descriptor, desc_origin) | + descriptor.isDescriptor() = false and + value = descriptor and origin = desc_origin + or + descriptor.isDescriptor() = true and + descriptor.descriptorGet(this, value, origin) + ) + } + } class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject { @@ -87,7 +101,7 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject ) } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + override predicate lookup(string name, ObjectInternal value, CfgOrigin origin) { exists(ClassObjectInternal decl | decl = Types::getMro(this).findDeclaringClass(name) | Types::declaredAttribute(decl, name, value, origin) @@ -142,7 +156,7 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + override predicate lookup(string name, ObjectInternal value, CfgOrigin origin) { value = ObjectInternal::fromBuiltin(this.getBuiltin().getMember(name)) and origin = CfgOrigin::unknown() } @@ -203,7 +217,7 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + override predicate lookup(string name, ObjectInternal value, CfgOrigin origin) { none() } @@ -251,7 +265,7 @@ class TypeInternal extends ClassObjectInternal, TType { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + override predicate lookup(string name, ObjectInternal value, CfgOrigin origin) { none() } diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 28d7128fba82..be0e7740af7b 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -91,7 +91,7 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { this = instance and descriptor.isDescriptor() = true and exists(AttrNode attr | PointsToInternal::pointsTo(attr.getObject(name), _, instance, _) and - this.getClass().attribute(name, descriptor, _) + this.getClass().(ClassObjectInternal).lookup(name, descriptor, _) ) } @@ -302,7 +302,7 @@ private predicate receiver_type(AttrNode attr, string name, ObjectInternal value } private predicate cls_descriptor(ClassObjectInternal cls, string name, ObjectInternal descriptor) { - cls.attribute(name, descriptor, _) and + cls.lookup(name, descriptor, _) and descriptor.isDescriptor() = true } From 9af70711313bf2fe22dbfde95dcb624397144732 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 9 Apr 2019 16:51:00 +0100 Subject: [PATCH 039/128] Python points-to: track return values from builtin methods. --- .../src/semmle/python/objects/Callables.qll | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 48240f252ede..89137b5f9462 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -274,9 +274,25 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } + pragma [noinline] override predicate callResult(ObjectInternal obj, CfgOrigin origin) { - // TO DO .. Result should be be a unknown value of a known class if the return type is known or just an unknown. - none() + exists(Builtin func, BuiltinClassObjectInternal cls | + func = this.getBuiltin() and + cls = ObjectInternal::fromBuiltin(this.getReturnType()) | + obj = TUnknownInstance(cls) + or + cls = ObjectInternal::noneType() and obj = ObjectInternal::none_() + or + cls = ObjectInternal::builtin("bool") and obj = ObjectInternal::bool(_) + ) and + origin = CfgOrigin::unknown() + } + + Builtin getReturnType() { + exists(Builtin func | + func = this.getBuiltin() | + ext_rettype(func, result) + ) } override ControlFlowNode getOrigin() { From 03159bb31c2c8a49ca4f366c7b353d350d1688bf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 9 Apr 2019 17:09:50 +0100 Subject: [PATCH 040/128] Python points-to: Track 'unknown' value through simple iteration. --- .../src/semmle/python/pointsto/PointsTo.qll | 9 ++- .../PointsTo/new/PointsToUnknown.expected | 66 +++++++++---------- .../PointsTo/new/PointsToUnknown.ql | 3 +- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 2f701649df27..d0d2b2a271ce 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -409,8 +409,8 @@ cached module PointsToInternal { scope_entry_points_to(def, context, value, origin) or InterModulePointsTo::implicit_submodule_points_to(def, context, value, origin) - // or - // iteration_definition_points_to(def, context, value, origin) + or + iteration_definition_points_to(def, context, value, origin) /* * No points-to for non-local function entry definitions yet. */ @@ -564,6 +564,11 @@ cached module PointsToInternal { ) } + private predicate iteration_definition_points_to(IterationDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + pointsTo(def.getSequence(), context, ObjectInternal::unknown(), _) and + value = ObjectInternal::unknown() and origin = def.getDefiningNode() + } + /** Holds if `f` is an expression node `tval if cond else fval` and points to `(value, origin)`. */ private predicate if_exp_points_to(IfExprNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { pointsTo(f.getAnOperand(), context, value, origin) diff --git a/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.expected b/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.expected index dd63fe12bb21..c8801ceabbc1 100644 --- a/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.expected +++ b/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.expected @@ -4,14 +4,15 @@ | a_simple.py:24 | ControlFlowNode for x | 23 | | a_simple.py:29 | ControlFlowNode for x | 27 | | a_simple.py:35 | ControlFlowNode for Subscript | 35 | +| a_simple.py:35 | ControlFlowNode for UnaryExpr | 35 | | a_simple.py:35 | ControlFlowNode for args | 34 | | a_simple.py:36 | ControlFlowNode for Subscript | 36 | +| a_simple.py:36 | ControlFlowNode for UnaryExpr | 36 | | a_simple.py:36 | ControlFlowNode for kwargs | 34 | | b_condition.py:5 | ControlFlowNode for IfExp | 5 | | b_condition.py:5 | ControlFlowNode for cond | 5 | | b_condition.py:5 | ControlFlowNode for unknown | 5 | | b_condition.py:5 | ControlFlowNode for unknown() | 5 | -| b_condition.py:5 | ControlFlowNode for x | 5 | | b_condition.py:7 | ControlFlowNode for x | 5 | | b_condition.py:9 | ControlFlowNode for use | 9 | | b_condition.py:9 | ControlFlowNode for use() | 9 | @@ -20,7 +21,6 @@ | b_condition.py:11 | ControlFlowNode for cond | 11 | | b_condition.py:11 | ControlFlowNode for unknown | 11 | | b_condition.py:11 | ControlFlowNode for unknown() | 11 | -| b_condition.py:11 | ControlFlowNode for x | 11 | | b_condition.py:13 | ControlFlowNode for x | 11 | | b_condition.py:15 | ControlFlowNode for use | 15 | | b_condition.py:15 | ControlFlowNode for use() | 15 | @@ -29,7 +29,7 @@ | b_condition.py:17 | ControlFlowNode for cond | 17 | | b_condition.py:17 | ControlFlowNode for unknown | 17 | | b_condition.py:17 | ControlFlowNode for unknown() | 17 | -| b_condition.py:17 | ControlFlowNode for x | 17 | +| b_condition.py:19 | ControlFlowNode for UnaryExpr | 19 | | b_condition.py:19 | ControlFlowNode for x | 17 | | b_condition.py:21 | ControlFlowNode for use | 21 | | b_condition.py:21 | ControlFlowNode for use() | 21 | @@ -38,7 +38,6 @@ | b_condition.py:23 | ControlFlowNode for cond | 23 | | b_condition.py:23 | ControlFlowNode for unknown | 23 | | b_condition.py:23 | ControlFlowNode for unknown() | 23 | -| b_condition.py:23 | ControlFlowNode for x | 23 | | b_condition.py:25 | ControlFlowNode for IfExp | 23 | | b_condition.py:25 | ControlFlowNode for x | 23 | | b_condition.py:26 | ControlFlowNode for use | 26 | @@ -53,7 +52,7 @@ | b_condition.py:31 | ControlFlowNode for cond | 31 | | b_condition.py:31 | ControlFlowNode for unknown | 31 | | b_condition.py:31 | ControlFlowNode for unknown() | 31 | -| b_condition.py:31 | ControlFlowNode for x | 31 | +| b_condition.py:32 | ControlFlowNode for UnaryExpr | 32 | | b_condition.py:32 | ControlFlowNode for x | 31 | | b_condition.py:34 | ControlFlowNode for use | 34 | | b_condition.py:34 | ControlFlowNode for use() | 34 | @@ -64,16 +63,14 @@ | b_condition.py:37 | ControlFlowNode for x | 31 | | b_condition.py:39 | ControlFlowNode for thing | 39 | | b_condition.py:39 | ControlFlowNode for thing() | 39 | -| b_condition.py:39 | ControlFlowNode for v2 | 39 | -| b_condition.py:41 | ControlFlowNode for Attribute | 39 | | b_condition.py:41 | ControlFlowNode for v2 | 39 | -| b_condition.py:42 | ControlFlowNode for Attribute | 39 | +| b_condition.py:42 | ControlFlowNode for Attribute | 42 | | b_condition.py:42 | ControlFlowNode for v2 | 39 | -| b_condition.py:43 | ControlFlowNode for Attribute | 39 | +| b_condition.py:43 | ControlFlowNode for Attribute | 43 | | b_condition.py:43 | ControlFlowNode for use | 43 | | b_condition.py:43 | ControlFlowNode for use() | 43 | | b_condition.py:43 | ControlFlowNode for v2 | 39 | -| b_condition.py:44 | ControlFlowNode for Attribute | 39 | +| b_condition.py:44 | ControlFlowNode for Attribute | 44 | | b_condition.py:44 | ControlFlowNode for use | 44 | | b_condition.py:44 | ControlFlowNode for use() | 44 | | b_condition.py:44 | ControlFlowNode for v2 | 39 | @@ -84,19 +81,19 @@ | b_condition.py:58 | ControlFlowNode for use | 58 | | b_condition.py:58 | ControlFlowNode for use() | 58 | | b_condition.py:58 | ControlFlowNode for v | 56 | -| b_condition.py:62 | ControlFlowNode for Attribute | 61 | +| b_condition.py:62 | ControlFlowNode for Attribute | 62 | | b_condition.py:62 | ControlFlowNode for x | 61 | | b_condition.py:64 | ControlFlowNode for y | 61 | -| b_condition.py:65 | ControlFlowNode for Attribute | 61 | +| b_condition.py:65 | ControlFlowNode for Attribute | 65 | | b_condition.py:65 | ControlFlowNode for x | 61 | -| b_condition.py:66 | ControlFlowNode for Attribute | 61 | +| b_condition.py:66 | ControlFlowNode for Attribute | 66 | | b_condition.py:66 | ControlFlowNode for seq | 66 | | b_condition.py:66 | ControlFlowNode for x | 61 | | b_condition.py:70 | ControlFlowNode for IfExp | 70 | -| b_condition.py:70 | ControlFlowNode for b | 70 | | b_condition.py:70 | ControlFlowNode for cond | 70 | | b_condition.py:70 | ControlFlowNode for unknown | 70 | | b_condition.py:70 | ControlFlowNode for unknown() | 70 | +| b_condition.py:71 | ControlFlowNode for UnaryExpr | 71 | | b_condition.py:71 | ControlFlowNode for b | 70 | | b_condition.py:73 | ControlFlowNode for b | 70 | | b_condition.py:79 | ControlFlowNode for use | 79 | @@ -120,70 +117,67 @@ | b_condition.py:99 | ControlFlowNode for use | 99 | | b_condition.py:99 | ControlFlowNode for use() | 99 | | b_condition.py:102 | ControlFlowNode for a | 101 | +| b_condition.py:104 | ControlFlowNode for UnaryExpr | 104 | | b_condition.py:104 | ControlFlowNode for a | 101 | | b_condition.py:105 | ControlFlowNode for Subscript | 105 | +| b_condition.py:105 | ControlFlowNode for UnaryExpr | 105 | | b_condition.py:105 | ControlFlowNode for a | 101 | | c_tests.py:5 | ControlFlowNode for IfExp | 5 | | c_tests.py:5 | ControlFlowNode for cond | 5 | | c_tests.py:5 | ControlFlowNode for unknown | 5 | | c_tests.py:5 | ControlFlowNode for unknown() | 5 | -| c_tests.py:5 | ControlFlowNode for x | 5 | | c_tests.py:7 | ControlFlowNode for x | 5 | | c_tests.py:10 | ControlFlowNode for cond | 10 | | c_tests.py:15 | ControlFlowNode for cond | 15 | | c_tests.py:21 | ControlFlowNode for cond | 21 | | c_tests.py:21 | ControlFlowNode for unknown | 21 | | c_tests.py:21 | ControlFlowNode for unknown() | 21 | -| c_tests.py:32 | ControlFlowNode for Attribute | 4 | -| c_tests.py:32 | ControlFlowNode for Attribute | 32 | | c_tests.py:32 | ControlFlowNode for IfExp | 32 | | c_tests.py:32 | ControlFlowNode for cond | 32 | | c_tests.py:32 | ControlFlowNode for unknown | 32 | | c_tests.py:32 | ControlFlowNode for unknown() | 32 | | c_tests.py:32 | ControlFlowNode for y | 4 | -| c_tests.py:34 | ControlFlowNode for Attribute | 4 | -| c_tests.py:34 | ControlFlowNode for Attribute | 32 | +| c_tests.py:34 | ControlFlowNode for Attribute | 34 | | c_tests.py:34 | ControlFlowNode for y | 4 | -| c_tests.py:37 | ControlFlowNode for Attribute | 4 | | c_tests.py:37 | ControlFlowNode for cond | 37 | | c_tests.py:37 | ControlFlowNode for y | 4 | -| c_tests.py:39 | ControlFlowNode for Attribute | 4 | +| c_tests.py:39 | ControlFlowNode for Attribute | 39 | | c_tests.py:39 | ControlFlowNode for y | 4 | -| c_tests.py:42 | ControlFlowNode for Attribute | 4 | | c_tests.py:42 | ControlFlowNode for cond | 42 | | c_tests.py:42 | ControlFlowNode for y | 4 | -| c_tests.py:44 | ControlFlowNode for Attribute | 4 | +| c_tests.py:44 | ControlFlowNode for Attribute | 44 | | c_tests.py:44 | ControlFlowNode for y | 4 | -| c_tests.py:48 | ControlFlowNode for Attribute | 4 | | c_tests.py:48 | ControlFlowNode for cond | 48 | | c_tests.py:48 | ControlFlowNode for unknown | 48 | | c_tests.py:48 | ControlFlowNode for unknown() | 48 | | c_tests.py:48 | ControlFlowNode for y | 4 | -| c_tests.py:50 | ControlFlowNode for Attribute | 4 | +| c_tests.py:50 | ControlFlowNode for Attribute | 50 | | c_tests.py:50 | ControlFlowNode for y | 4 | -| c_tests.py:53 | ControlFlowNode for Attribute | 4 | +| c_tests.py:53 | ControlFlowNode for Attribute | 53 | | c_tests.py:53 | ControlFlowNode for y | 4 | | c_tests.py:58 | ControlFlowNode for cond | 58 | | c_tests.py:63 | ControlFlowNode for cond | 63 | +| c_tests.py:65 | ControlFlowNode for hasattr() | 65 | | c_tests.py:73 | ControlFlowNode for x | 71 | | c_tests.py:73 | ControlFlowNode for y | 71 | +| c_tests.py:74 | ControlFlowNode for BinaryExpr | 74 | | c_tests.py:74 | ControlFlowNode for x | 71 | | c_tests.py:74 | ControlFlowNode for y | 71 | | c_tests.py:76 | ControlFlowNode for x | 71 | | c_tests.py:76 | ControlFlowNode for y | 71 | +| c_tests.py:77 | ControlFlowNode for BinaryExpr | 77 | | c_tests.py:77 | ControlFlowNode for x | 71 | | c_tests.py:77 | ControlFlowNode for y | 71 | | c_tests.py:80 | ControlFlowNode for IfExp | 80 | -| c_tests.py:80 | ControlFlowNode for b | 80 | | c_tests.py:80 | ControlFlowNode for cond | 80 | | c_tests.py:80 | ControlFlowNode for unknown | 80 | | c_tests.py:80 | ControlFlowNode for unknown() | 80 | | c_tests.py:81 | ControlFlowNode for b | 80 | | c_tests.py:83 | ControlFlowNode for IfExp | 83 | -| c_tests.py:83 | ControlFlowNode for b | 83 | | c_tests.py:83 | ControlFlowNode for cond | 83 | | c_tests.py:83 | ControlFlowNode for unknown | 83 | | c_tests.py:83 | ControlFlowNode for unknown() | 83 | +| c_tests.py:84 | ControlFlowNode for UnaryExpr | 84 | | c_tests.py:84 | ControlFlowNode for b | 83 | | c_tests.py:87 | ControlFlowNode for unknown | 87 | | c_tests.py:87 | ControlFlowNode for unknown() | 87 | @@ -191,13 +185,12 @@ | c_tests.py:90 | ControlFlowNode for cond | 90 | | c_tests.py:90 | ControlFlowNode for unknown | 90 | | c_tests.py:90 | ControlFlowNode for unknown() | 90 | -| c_tests.py:90 | ControlFlowNode for x | 90 | | c_tests.py:91 | ControlFlowNode for x | 90 | | c_tests.py:94 | ControlFlowNode for IfExp | 94 | | c_tests.py:94 | ControlFlowNode for cond | 94 | | c_tests.py:94 | ControlFlowNode for unknown | 94 | | c_tests.py:94 | ControlFlowNode for unknown() | 94 | -| c_tests.py:94 | ControlFlowNode for x | 94 | +| c_tests.py:95 | ControlFlowNode for UnaryExpr | 95 | | c_tests.py:95 | ControlFlowNode for x | 94 | | c_tests.py:99 | ControlFlowNode for bar | 99 | | c_tests.py:99 | ControlFlowNode for bar() | 99 | @@ -206,7 +199,6 @@ | c_tests.py:99 | ControlFlowNode for x | 98 | | c_tests.py:100 | ControlFlowNode for use | 100 | | c_tests.py:100 | ControlFlowNode for use() | 100 | -| c_tests.py:100 | ControlFlowNode for x | 98 | | h_classes.py:12 | ControlFlowNode for name | 12 | | h_classes.py:17 | ControlFlowNode for arg | 14 | | h_classes.py:18 | ControlFlowNode for name | 18 | @@ -214,22 +206,30 @@ | h_classes.py:28 | ControlFlowNode for choice | 25 | | h_classes.py:42 | ControlFlowNode for unknown | 42 | | h_classes.py:42 | ControlFlowNode for unknown() | 42 | +| r_regressions.py:9 | ControlFlowNode for Attribute | 9 | +| r_regressions.py:9 | ControlFlowNode for Attribute() | 9 | +| r_regressions.py:18 | ControlFlowNode for Attribute | 18 | +| r_regressions.py:18 | ControlFlowNode for Attribute() | 18 | +| r_regressions.py:20 | ControlFlowNode for Attribute | 20 | +| r_regressions.py:21 | ControlFlowNode for close | 20 | +| r_regressions.py:23 | ControlFlowNode for close | 20 | +| r_regressions.py:23 | ControlFlowNode for close() | 23 | | r_regressions.py:29 | ControlFlowNode for x | 27 | | r_regressions.py:31 | ControlFlowNode for y | 27 | | r_regressions.py:33 | ControlFlowNode for y | 27 | +| r_regressions.py:35 | ControlFlowNode for UnaryExpr | 35 | | r_regressions.py:36 | ControlFlowNode for z | 27 | | r_regressions.py:39 | ControlFlowNode for use | 39 | | r_regressions.py:39 | ControlFlowNode for use() | 39 | | r_regressions.py:39 | ControlFlowNode for y | 27 | -| r_regressions.py:43 | ControlFlowNode for List | 43 | | r_regressions.py:43 | ControlFlowNode for x | 43 | | r_regressions.py:43 | ControlFlowNode for x() | 43 | | r_regressions.py:52 | ControlFlowNode for msg | 51 | | r_regressions.py:64 | ControlFlowNode for do_validation | 64 | | r_regressions.py:64 | ControlFlowNode for do_validation() | 64 | +| r_regressions.py:73 | ControlFlowNode for setattr() | 73 | | r_regressions.py:90 | ControlFlowNode for Attribute | 90 | | r_regressions.py:90 | ControlFlowNode for Attribute() | 90 | | r_regressions.py:102 | ControlFlowNode for unrelated_call | 102 | | r_regressions.py:102 | ControlFlowNode for unrelated_call() | 102 | -| r_regressions.py:107 | ControlFlowNode for Attribute | 106 | | r_regressions.py:107 | ControlFlowNode for x | 106 | diff --git a/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.ql b/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.ql index e8258bc53a3f..9c9c432717f1 100644 --- a/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.ql +++ b/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.ql @@ -1,9 +1,10 @@ import python import Util import semmle.python.pointsto.PointsTo +import semmle.python.objects.ObjectInternal from ControlFlowNode f, ControlFlowNode x -where PointsTo::points_to(f, _, unknownValue(), _, x) +where PointsTo::pointsTo(f, _, ObjectInternal::unknown(), x) select locate(f.getLocation(), "abchr"), f.toString(), x.getLocation().getStartLine() From e21a863db9398330584fb712c179a47fed52fe02 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 9 Apr 2019 17:18:57 +0100 Subject: [PATCH 041/128] Python points-to: Update sanity check. --- python/ql/test/library-tests/PointsTo/new/Sanity.ql | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/python/ql/test/library-tests/PointsTo/new/Sanity.ql b/python/ql/test/library-tests/PointsTo/new/Sanity.ql index 496a751cd2b5..3f49d73d832b 100644 --- a/python/ql/test/library-tests/PointsTo/new/Sanity.ql +++ b/python/ql/test/library-tests/PointsTo/new/Sanity.ql @@ -1,6 +1,7 @@ import python import semmle.python.pointsto.PointsTo +import semmle.python.objects.ObjectInternal predicate ssa_sanity(string clsname, string problem, string what) { /* Exactly one definition of each SSA variable */ @@ -106,15 +107,6 @@ predicate ssa_sanity(string clsname, string problem, string what) { ) and problem = "does not have an ImplicitModuleNameDefinition" ) - or - // Unknown value should always have the class unknownType - exists(ControlFlowNode f, ClassObject cls | - PointsTo::points_to(f, _, unknownValue(), cls, _) and - clsname = f.getAQlClass() and - cls != theUnknownType() and - problem = "unknownValue() has class != theUnknownType()" and - what = cls.getName() - ) } from string clsname, string problem, string what From a442695ad054f6c4216232860d5d99d9764f4f76 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 10 Apr 2019 11:09:10 +0100 Subject: [PATCH 042/128] Python points-to: Improve handling of len(), sequences and comparisons. --- .../src/semmle/python/objects/Callables.qll | 6 ++ .../ql/src/semmle/python/objects/Classes.qll | 2 + .../src/semmle/python/objects/Constants.qll | 12 ++++ .../src/semmle/python/objects/Instances.qll | 19 ++++++ .../ql/src/semmle/python/objects/Modules.qll | 3 + .../semmle/python/objects/ObjectInternal.qll | 11 ++++ .../src/semmle/python/objects/Sequences.qll | 2 - python/ql/src/semmle/python/pointsto/Base.qll | 19 +++++- .../src/semmle/python/pointsto/PointsTo.qll | 61 ++++++++++++------- .../PointsTo/new/PointsToNone.expected | 4 +- .../PointsTo/new/PointsToUnknown.expected | 13 ++-- .../PointsTo/new/TestEvaluate.expected | 6 +- .../PointsTo/new/TestEvaluate.ql | 12 +++- .../PointsTo/new/code/c_tests.py | 14 ++--- 14 files changed, 137 insertions(+), 47 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 89137b5f9462..7c40e7779462 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -50,6 +50,8 @@ abstract class CallableObjectInternal extends ObjectInternal { abstract NameNode getParameterByName(string name); + override int length() { none() } + } @@ -474,6 +476,8 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { ) } + override int length() { none() } + } class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { @@ -532,6 +536,8 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + override int length() { none() } + } diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index ef4d01a8c445..ef47f92d6354 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -58,6 +58,8 @@ abstract class ClassObjectInternal extends ObjectInternal { ) } + override int length() { none() } + } class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject { diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index 62ee913826e2..b297dc995b62 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -62,6 +62,8 @@ abstract class BooleanObjectInternal extends ConstantObjectInternal { result = TBuiltinClassObject(Builtin::special("bool")) } + override int length() { none() } + } class TrueObjectInternal extends BooleanObjectInternal, TTrue { @@ -150,6 +152,8 @@ class NoneObjectInternal extends ConstantObjectInternal, TNone { none() } + override int length() { none() } + } @@ -186,6 +190,8 @@ class IntObjectInternal extends ConstantObjectInternal, TInt { this.intValue() != 0 and result = true } + override int length() { none() } + } class FloatObjectInternal extends ConstantObjectInternal, TFloat { @@ -229,6 +235,8 @@ class FloatObjectInternal extends ConstantObjectInternal, TFloat { this.floatValue() != 0.0 and result = true } + override int length() { none() } + } @@ -265,6 +273,10 @@ class StringObjectInternal extends ConstantObjectInternal, TString { this.strValue() != "" and result = true } + override int length() { + result = this.strValue().length() + } + } diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index be0e7740af7b..5c24570bfd95 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -95,6 +95,10 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { ) } + override int length() { + result = lengthFromClass(this.getClass()) + } + } bindingset[instance, mro, name] @@ -204,6 +208,10 @@ class SelfInstanceInternal extends TSelfInstance, ObjectInternal { instance = this } + override int length() { + result = lengthFromClass(this.getClass()) + } + } /** Represents a value that has a known class, but no other information */ @@ -294,8 +302,15 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { instance = this } + override int length() { + result = lengthFromClass(this.getClass()) + } + } +private int lengthFromClass(ClassObjectInternal cls) { + Types::getMro(cls).declares("__len__") and result = -1 +} private predicate receiver_type(AttrNode attr, string name, ObjectInternal value, ClassObjectInternal cls) { PointsToInternal::pointsTo(attr.getObject(name), _, value, _) and value.getClass() = cls @@ -379,5 +394,9 @@ class SuperInstance extends TSuperInstance, ObjectInternal { ) } + override int length() { + none() + } + } diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index e999b562faf3..897883d05d03 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -41,6 +41,9 @@ abstract class ModuleObjectInternal extends ObjectInternal { override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + + override int length() { none() } + } class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleObject { diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index d54fa6af3c0f..10b8f5f6e5fe 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -97,6 +97,11 @@ class ObjectInternal extends TObject { */ abstract predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor); + /** Gets the length of the sequence that this "object" represents. + * Always returns a value for a sequence, will be -1 if object has no fixed length. + */ + abstract int length(); + } @@ -168,6 +173,8 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + override int length() { none() } + } @@ -237,6 +244,8 @@ class UnknownInternal extends ObjectInternal, TUnknown { override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + override int length() { result = -1 } + } class UndefinedInternal extends ObjectInternal, TUndefined { @@ -307,6 +316,8 @@ class UndefinedInternal extends ObjectInternal, TUndefined { override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + override int length() { none() } + } module ObjectInternal { diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index 7776eac9df06..a0434cb0f2eb 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -15,8 +15,6 @@ abstract class SequenceObjectInternal extends ObjectInternal { abstract ObjectInternal getItem(int n); - abstract int length(); - /** The boolean value of this object, this may be both * true and false if the "object" represents a set of possible objects. */ override boolean booleanValue() { diff --git a/python/ql/src/semmle/python/pointsto/Base.qll b/python/ql/src/semmle/python/pointsto/Base.qll index 7bbba90979df..74d68b3fc42d 100644 --- a/python/ql/src/semmle/python/pointsto/Base.qll +++ b/python/ql/src/semmle/python/pointsto/Base.qll @@ -268,7 +268,24 @@ class ParameterDefinition extends PyNodeDefinition { } ControlFlowNode getDefault() { - result.getNode() = this.getParameter().getDefault() + exists(Function f, int n, int c, int d, Arguments args | + args = f.getDefinition().getArgs() | + f.getArg(n) = this.getDefiningNode().getNode() and + c = count(f.getAnArg()) and + d = count(args.getADefault()) and + result.getNode() = args.getDefault(d-c+n) + ) + } + + predicate isVarargs() { + exists(Function func | func.getVararg() = this.getDefiningNode().getNode()) + } + + /** Holds if this parameter is a 'kwargs' parameter. + * The `kwargs` in `f(a, b, **kwargs)`. + */ + predicate isKwargs() { + exists(Function func | func.getKwarg() = this.getDefiningNode().getNode()) } Parameter getParameter() { diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index d0d2b2a271ce..f523fa44d03d 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -840,7 +840,7 @@ module InterProceduralPointsTo { callsite_argument_transfer(arg, caller, def, context) ) or - not def.isSelf() and not def.getParameter().isVarargs() and not def.getParameter().isKwargs() and + not def.isSelf() and not def.isVarargs() and not def.isKwargs() and context.isRuntime() and value = ObjectInternal::unknown() and origin = def.getDefiningNode() } @@ -881,7 +881,7 @@ module InterProceduralPointsTo { context.fromCall(call, func, caller) and func.getScope().getArg(n) = def.getParameter() and not exists(call.getArg(n)) and - not exists(call.getArgByName(def.getParameter().asName().getId())) and + not exists(call.getArgByName(def.getVariable().getName())) and not exists(call.getNode().getKwargs()) and not exists(call.getNode().getStarargs()) ) @@ -890,25 +890,24 @@ module InterProceduralPointsTo { /** Helper for parameter_points_to */ pragma [noinline] private predicate special_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - context.isRuntime() and - origin = def.getDefiningNode() and - exists(ControlFlowNode param | - param = def.getDefiningNode() | - exists(Function func | func.getVararg() = param.getNode()) and value = TUnknownInstance(ObjectInternal::builtin("tuple")) + ( + def.isVarargs() and value = TUnknownInstance(ObjectInternal::builtin("tuple")) or - exists(Function func | func.getKwarg() = param.getNode()) and value = TUnknownInstance(ObjectInternal::builtin("dict")) - ) - or - exists(PointsToContext caller, CallNode call, Function f, Parameter p | - context.fromCall(call, caller) and - context.appliesToScope(f) and - f.getAnArg() = p and p = def.getParameter() and - not p.isSelf() and - not exists(call.getArg(p.getPosition())) and - not exists(call.getArgByName(p.getName())) and - (exists(call.getNode().getKwargs()) or exists(call.getNode().getStarargs())) and - value = ObjectInternal::unknown() and origin = def.getDefiningNode() + def.isKwargs() and value = TUnknownInstance(ObjectInternal::builtin("dict")) ) + and + ( + context.isRuntime() + or + exists(PointsToContext caller, CallNode call, Parameter p | + context.fromCall(call, caller) and + context.appliesToScope(def.getScope()) and + p = def.getParameter() and + not exists(call.getArg(p.getPosition())) and + not exists(call.getArgByName(p.getName())) + ) + ) and + origin = def.getDefiningNode() } /** Holds if the `(argument, caller)` pair matches up with `(param, callee)` pair across call. */ @@ -1093,18 +1092,30 @@ module Expressions { arg = call.getArg(0) and exists(BuiltinFunctionObjectInternal callable | PointsToInternal::pointsTo(call.getFunction(), context, callable, _) | - callable = ObjectInternal::builtin("len") and value = TInt(argvalue.(SequenceObjectInternal).length()) - or callable != ObjectInternal::builtin("len") and callable != ObjectInternal::builtin("callable") and callable != ObjectInternal::builtin("isinstance") and callable != ObjectInternal::builtin("issubclass") and + callable != ObjectInternal::builtin("hasattr") and callable.isClass() = false and value = ObjectInternal::unknown() ) and origin = call } + pragma [noinline] + private predicate lenCallPointsTo(CallNode call, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode arg, ObjectInternal argvalue) { + len_call(call, arg, context, argvalue) and + origin = call and + exists(int len | + len = argvalue.length() + | + value = TInt(len) and len >= 0 + or + len < 0 and value = TUnknownInstance(ObjectInternal::builtin("int")) + ) + } + private boolean otherComparisonEvaluatesTo(CompareNode comp, PointsToContext context, ControlFlowNode operand, ObjectInternal opvalue) { exists(Cmpop op | comp.operands(operand, op, _) and @@ -1204,6 +1215,8 @@ module Expressions { or builtinCallPointsTo(expr, context, value, origin, subexpr, subvalue) or + lenCallPointsTo(expr, context, value, origin, subexpr, subvalue) + or value = ObjectInternal::bool(evaluatesTo(expr, context, subexpr, subvalue)) and origin = expr } @@ -1274,6 +1287,12 @@ module Expressions { PointsToInternal::pointsTo(use, context, val, _) } + private predicate len_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { + PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("len"), _) and + use = call.getArg(0) and + PointsToInternal::pointsTo(use, context, val, _) + } + private predicate hasattr_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, string name) { exists(ControlFlowNode func, ControlFlowNode arg1 | call2(call, func, use, arg1) and diff --git a/python/ql/test/library-tests/PointsTo/new/PointsToNone.expected b/python/ql/test/library-tests/PointsTo/new/PointsToNone.expected index f5f838edd8a5..a72036844ab9 100644 --- a/python/ql/test/library-tests/PointsTo/new/PointsToNone.expected +++ b/python/ql/test/library-tests/PointsTo/new/PointsToNone.expected @@ -37,6 +37,7 @@ | c_tests.py:5 | ControlFlowNode for x | 5 | | c_tests.py:7 | ControlFlowNode for None | 7 | | c_tests.py:7 | ControlFlowNode for x | 5 | +| c_tests.py:8 | ControlFlowNode for x | 5 | | c_tests.py:32 | ControlFlowNode for Attribute | 32 | | c_tests.py:32 | ControlFlowNode for IfExp | 32 | | c_tests.py:32 | ControlFlowNode for None | 32 | @@ -86,9 +87,6 @@ | r_regressions.py:9 | ControlFlowNode for Attribute() | 11 | | r_regressions.py:13 | ControlFlowNode for Attribute | 13 | | r_regressions.py:13 | ControlFlowNode for None | 13 | -| r_regressions.py:20 | ControlFlowNode for Attribute | 13 | -| r_regressions.py:20 | ControlFlowNode for close | 13 | -| r_regressions.py:21 | ControlFlowNode for close | 13 | | r_regressions.py:22 | ControlFlowNode for Attribute | 22 | | r_regressions.py:22 | ControlFlowNode for None | 22 | | r_regressions.py:27 | ControlFlowNode for None | 27 | diff --git a/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.expected b/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.expected index c8801ceabbc1..c715512184f9 100644 --- a/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.expected +++ b/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.expected @@ -1,14 +1,10 @@ -| a_simple.py:15 | ControlFlowNode for t | 14 | -| a_simple.py:16 | ControlFlowNode for d | 14 | | a_simple.py:20 | ControlFlowNode for seq | 18 | | a_simple.py:24 | ControlFlowNode for x | 23 | | a_simple.py:29 | ControlFlowNode for x | 27 | | a_simple.py:35 | ControlFlowNode for Subscript | 35 | | a_simple.py:35 | ControlFlowNode for UnaryExpr | 35 | -| a_simple.py:35 | ControlFlowNode for args | 34 | | a_simple.py:36 | ControlFlowNode for Subscript | 36 | | a_simple.py:36 | ControlFlowNode for UnaryExpr | 36 | -| a_simple.py:36 | ControlFlowNode for kwargs | 34 | | b_condition.py:5 | ControlFlowNode for IfExp | 5 | | b_condition.py:5 | ControlFlowNode for cond | 5 | | b_condition.py:5 | ControlFlowNode for unknown | 5 | @@ -116,17 +112,14 @@ | b_condition.py:97 | ControlFlowNode for x | 87 | | b_condition.py:99 | ControlFlowNode for use | 99 | | b_condition.py:99 | ControlFlowNode for use() | 99 | -| b_condition.py:102 | ControlFlowNode for a | 101 | -| b_condition.py:104 | ControlFlowNode for UnaryExpr | 104 | -| b_condition.py:104 | ControlFlowNode for a | 101 | | b_condition.py:105 | ControlFlowNode for Subscript | 105 | | b_condition.py:105 | ControlFlowNode for UnaryExpr | 105 | -| b_condition.py:105 | ControlFlowNode for a | 101 | | c_tests.py:5 | ControlFlowNode for IfExp | 5 | | c_tests.py:5 | ControlFlowNode for cond | 5 | | c_tests.py:5 | ControlFlowNode for unknown | 5 | | c_tests.py:5 | ControlFlowNode for unknown() | 5 | | c_tests.py:7 | ControlFlowNode for x | 5 | +| c_tests.py:8 | ControlFlowNode for x | 5 | | c_tests.py:10 | ControlFlowNode for cond | 10 | | c_tests.py:15 | ControlFlowNode for cond | 15 | | c_tests.py:21 | ControlFlowNode for cond | 21 | @@ -157,7 +150,6 @@ | c_tests.py:53 | ControlFlowNode for y | 4 | | c_tests.py:58 | ControlFlowNode for cond | 58 | | c_tests.py:63 | ControlFlowNode for cond | 63 | -| c_tests.py:65 | ControlFlowNode for hasattr() | 65 | | c_tests.py:73 | ControlFlowNode for x | 71 | | c_tests.py:73 | ControlFlowNode for y | 71 | | c_tests.py:74 | ControlFlowNode for BinaryExpr | 74 | @@ -173,12 +165,14 @@ | c_tests.py:80 | ControlFlowNode for unknown | 80 | | c_tests.py:80 | ControlFlowNode for unknown() | 80 | | c_tests.py:81 | ControlFlowNode for b | 80 | +| c_tests.py:82 | ControlFlowNode for b | 80 | | c_tests.py:83 | ControlFlowNode for IfExp | 83 | | c_tests.py:83 | ControlFlowNode for cond | 83 | | c_tests.py:83 | ControlFlowNode for unknown | 83 | | c_tests.py:83 | ControlFlowNode for unknown() | 83 | | c_tests.py:84 | ControlFlowNode for UnaryExpr | 84 | | c_tests.py:84 | ControlFlowNode for b | 83 | +| c_tests.py:85 | ControlFlowNode for b | 83 | | c_tests.py:87 | ControlFlowNode for unknown | 87 | | c_tests.py:87 | ControlFlowNode for unknown() | 87 | | c_tests.py:90 | ControlFlowNode for IfExp | 90 | @@ -186,6 +180,7 @@ | c_tests.py:90 | ControlFlowNode for unknown | 90 | | c_tests.py:90 | ControlFlowNode for unknown() | 90 | | c_tests.py:91 | ControlFlowNode for x | 90 | +| c_tests.py:92 | ControlFlowNode for x | 90 | | c_tests.py:94 | ControlFlowNode for IfExp | 94 | | c_tests.py:94 | ControlFlowNode for cond | 94 | | c_tests.py:94 | ControlFlowNode for unknown | 94 | diff --git a/python/ql/test/library-tests/PointsTo/new/TestEvaluate.expected b/python/ql/test/library-tests/PointsTo/new/TestEvaluate.expected index 9bdee5cf24e1..a84466ea16a6 100644 --- a/python/ql/test/library-tests/PointsTo/new/TestEvaluate.expected +++ b/python/ql/test/library-tests/PointsTo/new/TestEvaluate.expected @@ -6,7 +6,6 @@ | b_condition.py:36 | isinstance() | true | x | int 1 | | b_condition.py:36 | isinstance() | true | x | int 7 | | b_condition.py:71 | UnaryExpr | false | b | bool True | -| b_condition.py:77 | Compare | true | object | builtin-class object | | b_condition.py:77 | Compare | true | t | builtin-class type | | b_condition.py:82 | callable() | false | foo | bool True | | b_condition.py:88 | x | false | x | NoneType None | @@ -17,13 +16,18 @@ | b_condition.py:96 | y | false | y | NoneType None | | b_condition.py:102 | UnaryExpr | false | a | a | | b_condition.py:104 | UnaryExpr | false | a | a | +| b_condition.py:104 | UnaryExpr | true | a | a | +| b_condition.py:105 | UnaryExpr | false | a | a | +| b_condition.py:105 | UnaryExpr | true | a | a | | c_tests.py:7 | Compare | true | x | NoneType None | | c_tests.py:12 | x | false | x | int 0 | | c_tests.py:12 | x | true | x | int 1 | | c_tests.py:17 | Compare | false | x | int 1 | | c_tests.py:17 | Compare | true | x | int 0 | +| c_tests.py:23 | len() | false | x | List | | c_tests.py:23 | len() | true | x | List | | c_tests.py:23 | len() | true | x | Tuple | +| c_tests.py:26 | Compare | false | x | List | | c_tests.py:26 | Compare | false | x | Tuple | | c_tests.py:26 | Compare | true | x | List | | c_tests.py:26 | Compare | true | x | Tuple | diff --git a/python/ql/test/library-tests/PointsTo/new/TestEvaluate.ql b/python/ql/test/library-tests/PointsTo/new/TestEvaluate.ql index 2256d6910f13..7607cd23eca4 100644 --- a/python/ql/test/library-tests/PointsTo/new/TestEvaluate.ql +++ b/python/ql/test/library-tests/PointsTo/new/TestEvaluate.ql @@ -5,9 +5,15 @@ import semmle.python.pointsto.PointsToContext import Util -from ControlFlowNode test, ControlFlowNode use, Value val, boolean eval, PointsToContext ctx +from ControlFlowNode test, ControlFlowNode use, Value val, boolean eval, PointsToContext ctx, ControlFlowNode origin, string what where not use instanceof NameConstantNode and not use.getNode() instanceof ImmutableLiteral and -eval = Conditionals::testEvaluates(test, use, ctx, val, _) -select locate(test.getLocation(), "bc"), test.getNode().toString(), eval.toString(), use.getNode().toString(), val.getSource().toString() +eval = Conditionals::testEvaluates(test, use, ctx, val, origin) and +( + what = val.getSource().toString() + or + not exists(val.getSource()) and what = origin.getNode().toString() +) +select locate(test.getLocation(), "bc"), test.getNode().toString(), eval.toString(), use.getNode().toString(), what + diff --git a/python/ql/test/library-tests/PointsTo/new/code/c_tests.py b/python/ql/test/library-tests/PointsTo/new/code/c_tests.py index 206157c0c686..98479d933cbb 100644 --- a/python/ql/test/library-tests/PointsTo/new/code/c_tests.py +++ b/python/ql/test/library-tests/PointsTo/new/code/c_tests.py @@ -5,17 +5,17 @@ def f(y): x = unknown() if cond else None if x is None: - pass + x x = 0 if cond else 1 if x: - pass + x x = 0 if cond else 1 if x == 0: - pass + x x = ((1,2) if cond else (1,2,3)) if unknown() else [1,2] @@ -58,7 +58,7 @@ def others(x): x = bool if cond else type if issubclass(x, int): - pass + x x = 0 if cond else float @@ -79,17 +79,17 @@ def compound(x=1, y=0): def h(): b = unknown() if cond else True if b: - pass + b b = unknown() if cond else True if not b: - pass + b if unknown() == 3: pass x = unknown() if cond else None if x: - pass + x x = unknown() if cond else None if not x: From e82b2c422e89099039b5dbb1f207ab1988f5aacd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 10 Apr 2019 11:18:39 +0100 Subject: [PATCH 043/128] Python: Update test to use new API. --- python/ql/test/library-tests/PointsTo/new/Reachable.ql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ql/test/library-tests/PointsTo/new/Reachable.ql b/python/ql/test/library-tests/PointsTo/new/Reachable.ql index 60fccc308ee8..577d724c8e67 100644 --- a/python/ql/test/library-tests/PointsTo/new/Reachable.ql +++ b/python/ql/test/library-tests/PointsTo/new/Reachable.ql @@ -4,5 +4,5 @@ private import semmle.python.pointsto.PointsTo import Util from ControlFlowNode f, Context ctx -where PointsTo::Test::reachableBlock(f.getBasicBlock(), ctx) +where PointsToInternal::reachableBlock(f.getBasicBlock(), ctx) select locate(f.getLocation(), "m"), f.toString(), ctx From 4a03fd03cd4923fe363256eab42d9791dd678ccd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 10 Apr 2019 11:52:32 +0100 Subject: [PATCH 044/128] Update test to reflect new true positive. --- python/ql/test/query-tests/Security/CWE-215/FlaskDebug.expected | 1 + python/ql/test/query-tests/Security/CWE-215/test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ql/test/query-tests/Security/CWE-215/FlaskDebug.expected b/python/ql/test/query-tests/Security/CWE-215/FlaskDebug.expected index 3a23ea54ee29..a8ae2994cb6e 100644 --- a/python/ql/test/query-tests/Security/CWE-215/FlaskDebug.expected +++ b/python/ql/test/query-tests/Security/CWE-215/FlaskDebug.expected @@ -1,3 +1,4 @@ | test.py:10:1:10:19 | ControlFlowNode for Attribute() | A Flask app appears to be run in debug mode. This may allow an attacker to run arbitrary code through the debugger. | | test.py:25:1:25:20 | ControlFlowNode for Attribute() | A Flask app appears to be run in debug mode. This may allow an attacker to run arbitrary code through the debugger. | | test.py:29:1:29:20 | ControlFlowNode for Attribute() | A Flask app appears to be run in debug mode. This may allow an attacker to run arbitrary code through the debugger. | +| test.py:37:1:37:18 | ControlFlowNode for runapp() | A Flask app appears to be run in debug mode. This may allow an attacker to run arbitrary code through the debugger. | diff --git a/python/ql/test/query-tests/Security/CWE-215/test.py b/python/ql/test/query-tests/Security/CWE-215/test.py index 6054caf1d33b..73913b6b3b88 100644 --- a/python/ql/test/query-tests/Security/CWE-215/test.py +++ b/python/ql/test/query-tests/Security/CWE-215/test.py @@ -31,7 +31,7 @@ def main(): if False: app.run(debug=True) -# false negative + runapp = app.run runapp(debug=True) From 53f859117e164657679870b2de6b59e645ebd9c0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Apr 2019 10:14:27 +0100 Subject: [PATCH 045/128] Python points-to: Fix attribute lookup for packages. --- python/ql/src/semmle/python/types/ModuleObject.qll | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/ql/src/semmle/python/types/ModuleObject.qll b/python/ql/src/semmle/python/types/ModuleObject.qll index a9d848839f6a..105f56467adb 100644 --- a/python/ql/src/semmle/python/types/ModuleObject.qll +++ b/python/ql/src/semmle/python/types/ModuleObject.qll @@ -6,7 +6,11 @@ private import semmle.python.types.ModuleKind abstract class ModuleObject extends Object { ModuleValue theModule() { - result.getSource() = this + result.(PythonModuleObjectInternal).getSourceModule() = this.getModule() + or + result.(PackageObjectInternal).getFolder() = this.(PackageObject).getPath() + or + result.(BuiltinModuleObjectInternal).getBuiltin() = this } /** Gets the scope corresponding to this module, if this is a Python module */ @@ -223,7 +227,7 @@ class PackageObject extends ModuleObject { override Object getAttribute(string name) { exists(Value val | - theModule().(PackageObjectInternal).attribute(name, _, _) and + theModule().(PackageObjectInternal).attribute(name, val, _) and result = val.getSource() ) } From 3b42f3cea3a8498f029645dd1dffafa1573873a7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Apr 2019 10:32:27 +0100 Subject: [PATCH 046/128] Python points-to/taint-tracking: Fix up flow into __init__ methods. --- python/ql/src/semmle/python/objects/Classes.qll | 5 ++--- python/ql/src/semmle/python/objects/ObjectAPI.qll | 4 ++++ python/ql/src/semmle/python/security/TaintTracking.qll | 6 +++--- python/ql/src/semmle/python/types/ClassObject.qll | 4 ++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index ef47f92d6354..b9f99ce37a13 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -97,9 +97,8 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject override predicate calleeAndOffset(Function scope, int paramOffset) { exists(PythonFunctionObjectInternal init | - // TO DO... Lookup init... - none() | - init.getScope() = scope and paramOffset = 1 + this.lookup("__init__", init, _) and + init.calleeAndOffset(scope, paramOffset-1) ) } diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index abc32b187e19..6eb8f97f4f0f 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -146,5 +146,9 @@ class ClassValue extends Value { result = Types::getMro(this).getAnItem() } + Value lookup(string name) { + this.(ClassObjectInternal).lookup(name, result, _) + } + } diff --git a/python/ql/src/semmle/python/security/TaintTracking.qll b/python/ql/src/semmle/python/security/TaintTracking.qll index fff65b988d5b..3cae6ea5e2a6 100755 --- a/python/ql/src/semmle/python/security/TaintTracking.qll +++ b/python/ql/src/semmle/python/security/TaintTracking.qll @@ -1069,7 +1069,7 @@ library module TaintFlowImplementation { predicate self_init_end_transfer(EssaVariable self, CallContext callee, CallNode call, CallContext caller) { exists(ClassValue cls, Function init | call.getFunction().pointsTo(cls) and - init = cls.attr("__init__").(CallableValue).getScope() and + init = cls.lookup("__init__").(CallableValue).getScope() and self.getSourceVariable().(Variable).isSelf() and self.getScope() = init | callee = caller.getCallee(call) @@ -1218,7 +1218,7 @@ library module TaintFlowImplementation { pragma [noinline] predicate class_initializer_argument(ClassValue cls, int n, CallNode call, CallableValue func, ControlFlowNode argument, NameNode param) { call.getFunction().pointsTo(cls) and - cls.attr("__init__") = func and + cls.lookup("__init__") = func and call.getArg(n) = argument and param.getNode() = func.getScope().getArg(n+1) } @@ -1510,7 +1510,7 @@ class CallContext extends TCallContext { exists(ClassValue cls,CallNode call | this = TCalleeContext(call, _, _) and call.getFunction().pointsTo(cls) and - s = cls.attr("__init__").(CallableValue).getScope() and + s = cls.lookup("__init__").(CallableValue).getScope() and call.getFunction().pointsTo(cls) ) } diff --git a/python/ql/src/semmle/python/types/ClassObject.qll b/python/ql/src/semmle/python/types/ClassObject.qll index 6de3ca3a27d5..acc7582e3d16 100644 --- a/python/ql/src/semmle/python/types/ClassObject.qll +++ b/python/ql/src/semmle/python/types/ClassObject.qll @@ -108,7 +108,7 @@ class ClassObject extends Object { Will include attributes of super-classes */ Object lookupAttribute(string name) { exists(Value val | - theClass().attribute(name, val, _) and + theClass().lookup(name, val, _) and result = val.getSource() ) } @@ -135,7 +135,7 @@ class ClassObject extends Object { /** Whether the named attribute refers to the object, class and origin */ predicate attributeRefersTo(string name, Object obj, ClassObject cls, ControlFlowNode origin) { exists(Value val, CfgOrigin valorig | - theClass().attribute(name, val, valorig) and + theClass().lookup(name, val, valorig) and obj = val.getSource() and cls = val.getClass().getSource() and origin = valorig.toCfgNode() From ec080419ba79ed42b24ea5d5d4d4ac082a609841 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Apr 2019 18:08:13 +0100 Subject: [PATCH 047/128] Python points-to: Assorted tweaks to get nore tests passing. --- .../ql/src/Exceptions/UnguardedNextInGenerator.ql | 2 +- python/ql/src/analysis/CallGraphEfficiency.ql | 8 ++++---- .../ql/src/analysis/CallGraphMarginalEfficiency.ql | 10 +++++----- python/ql/src/analysis/KeyPointsToFailure.ql | 2 +- python/ql/src/analysis/Pruned.ql | 2 +- python/ql/src/semmle/python/pointsto/Filters.qll | 13 +++++++++++++ python/ql/src/semmle/python/pointsto/PointsTo.qll | 14 -------------- .../ql/src/semmle/python/types/FunctionObject.qll | 4 ++++ python/ql/src/semmle/python/types/Object.qll | 9 +++++++++ 9 files changed, 38 insertions(+), 26 deletions(-) diff --git a/python/ql/src/Exceptions/UnguardedNextInGenerator.ql b/python/ql/src/Exceptions/UnguardedNextInGenerator.ql index 77f4f9bc6b0d..81680c6d5c17 100755 --- a/python/ql/src/Exceptions/UnguardedNextInGenerator.ql +++ b/python/ql/src/Exceptions/UnguardedNextInGenerator.ql @@ -16,7 +16,7 @@ FunctionObject iter() { result = Object::builtin("iter") } -FunctionObject next() { +BuiltinFunctionObject next() { result = Object::builtin("next") } diff --git a/python/ql/src/analysis/CallGraphEfficiency.ql b/python/ql/src/analysis/CallGraphEfficiency.ql index f15565687334..8db6ae55ccf6 100644 --- a/python/ql/src/analysis/CallGraphEfficiency.ql +++ b/python/ql/src/analysis/CallGraphEfficiency.ql @@ -9,15 +9,15 @@ import semmle.python.pointsto.PointsToContext from int total_facts, int total_size, int depth, float efficiency where -total_facts = strictcount(ControlFlowNode call, FunctionObject func | +total_facts = strictcount(ControlFlowNode call, CallableValue func | exists(PointsToContext ctx | - call = PointsTo::get_a_call(func, ctx) and + call = func.getACall(ctx) and depth = ctx.getDepth() ) ) and -total_size = strictcount(ControlFlowNode call, FunctionObject func, PointsToContext ctx | - call = PointsTo::get_a_call(func, ctx) and +total_size = strictcount(ControlFlowNode call, CallableValue func, PointsToContext ctx | + call = func.getACall(ctx) and depth = ctx.getDepth() ) and diff --git a/python/ql/src/analysis/CallGraphMarginalEfficiency.ql b/python/ql/src/analysis/CallGraphMarginalEfficiency.ql index 72ca0383d5c8..e69ad954d22e 100644 --- a/python/ql/src/analysis/CallGraphMarginalEfficiency.ql +++ b/python/ql/src/analysis/CallGraphMarginalEfficiency.ql @@ -8,20 +8,20 @@ import semmle.python.pointsto.PointsToContext from int total_facts, int total_size, int depth, float efficiency where -total_facts = strictcount(ControlFlowNode call, FunctionObject func | +total_facts = strictcount(ControlFlowNode call, CallableValue func | exists(PointsToContext ctx | - call = PointsTo::get_a_call(func, ctx) and + call = func.getACall(ctx) and depth = ctx.getDepth() and not exists(PointsToContext shallower | - call = PointsTo::get_a_call(func, shallower) and + call = func.getACall(shallower) and shallower.getDepth() < depth ) ) ) and -total_size = strictcount(ControlFlowNode call, FunctionObject func, PointsToContext ctx | - call = PointsTo::get_a_call(func, ctx) and +total_size = strictcount(ControlFlowNode call, CallableValue func, PointsToContext ctx | + call = func.getACall(ctx) and depth = ctx.getDepth() ) and diff --git a/python/ql/src/analysis/KeyPointsToFailure.ql b/python/ql/src/analysis/KeyPointsToFailure.ql index 0139586aca84..79439134a070 100644 --- a/python/ql/src/analysis/KeyPointsToFailure.ql +++ b/python/ql/src/analysis/KeyPointsToFailure.ql @@ -12,7 +12,7 @@ import python predicate points_to_failure(Expr e) { exists(ControlFlowNode f | f = e.getAFlowNode() | - not f.refersTo(_) + not exists(f.pointsTo()) ) } diff --git a/python/ql/src/analysis/Pruned.ql b/python/ql/src/analysis/Pruned.ql index a40d47949e5c..d94bef177ab6 100644 --- a/python/ql/src/analysis/Pruned.ql +++ b/python/ql/src/analysis/Pruned.ql @@ -6,7 +6,7 @@ from int size where size = count(ControlFlowNode f | - not PointsTo::Test::reachableBlock(f.getBasicBlock(), _) + not PointsToInternal::reachableBlock(f.getBasicBlock(), _) ) diff --git a/python/ql/src/semmle/python/pointsto/Filters.qll b/python/ql/src/semmle/python/pointsto/Filters.qll index c69b72b70965..c088ff895a38 100644 --- a/python/ql/src/semmle/python/pointsto/Filters.qll +++ b/python/ql/src/semmle/python/pointsto/Filters.qll @@ -31,3 +31,16 @@ predicate issubclass(CallNode fc, ControlFlowNode cls, ControlFlowNode use) { fc.getArg(0) = use and cls = fc.getArg(1) } +/** Holds if `c` is a test comparing `x` and `y`. `is` is true if the operator is `is` or `==`, it is false if the operator is `is not` or `!=`. */ +predicate equality_test(CompareNode c, ControlFlowNode x, boolean is, ControlFlowNode y) { + exists(Cmpop op | + c.operands(x, op, y) or + c.operands(y, op, x) + | + (is = true and op instanceof Is or + is = false and op instanceof IsNot or + is = true and op instanceof Eq or + is = false and op instanceof NotEq + ) + ) +} diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index f523fa44d03d..91cdf36f814e 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1145,20 +1145,6 @@ module Expressions { ) } - /** Holds if `c` is a test comparing `x` and `y`. `is` is true if the operator is `is` or `==`, it is false if the operator is `is not` or `!=`. */ - private predicate equality_test(CompareNode c, ControlFlowNode x, boolean is, ControlFlowNode y) { - exists(Cmpop op | - c.operands(x, op, y) or - c.operands(y, op, x) - | - (is = true and op instanceof Is or - is = false and op instanceof IsNot or - is = true and op instanceof Eq or - is = false and op instanceof NotEq - ) - ) - } - pragma [noinline] private boolean inequalityEvaluatesTo(ControlFlowNode expr, PointsToContext context, ControlFlowNode use, ObjectInternal val) { exists(ControlFlowNode r, boolean sense | diff --git a/python/ql/src/semmle/python/types/FunctionObject.qll b/python/ql/src/semmle/python/types/FunctionObject.qll index 4087afc89420..37f7f506422d 100644 --- a/python/ql/src/semmle/python/types/FunctionObject.qll +++ b/python/ql/src/semmle/python/types/FunctionObject.qll @@ -257,6 +257,10 @@ abstract class BuiltinCallable extends FunctionObject { abstract override string getQualifiedName(); + override ControlFlowNode getArgumentForCall(CallNode call, int n) { + call = this.getACall() and result = call.getArg(n) + } + } class BuiltinMethodObject extends BuiltinCallable { diff --git a/python/ql/src/semmle/python/types/Object.qll b/python/ql/src/semmle/python/types/Object.qll index 200bc16e001d..e60830d89c83 100644 --- a/python/ql/src/semmle/python/types/Object.qll +++ b/python/ql/src/semmle/python/types/Object.qll @@ -515,6 +515,15 @@ class SuperBoundMethod extends Object { result = "super()." + name } + Object getFunction(string fname) { + fname = name and + exists(SuperInstance sup, BoundMethodObjectInternal m | + sup = this.(AttrNode).getObject(name).pointsTo() and + sup.attribute(name, m, _) and + result = m.getFunction().getSource() + ) + } + } From 62e05187af722a0584ba3096823ec45bd6c14d4f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 12 Apr 2019 10:27:46 +0100 Subject: [PATCH 048/128] Python points-to: Add property objects. --- .../src/semmle/python/objects/Callables.qll | 132 ----------- .../src/semmle/python/objects/Descriptors.qll | 218 ++++++++++++++++++ .../src/semmle/python/objects/Instances.qll | 2 +- .../semmle/python/objects/ObjectInternal.qll | 4 + .../ql/src/semmle/python/objects/TObject.qll | 5 + 5 files changed, 228 insertions(+), 133 deletions(-) create mode 100644 python/ql/src/semmle/python/objects/Descriptors.qll diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 7c40e7779462..edcace21ede2 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -407,138 +407,6 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { } -class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { - - override string toString() { - result = "classmethod(" + this.getFunction() + ")" - } - - override boolean booleanValue() { result = true } - - override predicate introduced(ControlFlowNode node, PointsToContext context) { - exists(CallableObjectInternal function | - this = TClassMethod(node, function) and - class_method(node, function, context) - ) - } - - CallableObjectInternal getFunction() { - this = TClassMethod(_, result) - } - - override ClassDecl getClassDeclaration() { none() } - - override boolean isClass() { result = false } - - override ObjectInternal getClass() { result = ObjectInternal::classMethod() } - - override boolean isComparable() { none() } - - override Builtin getBuiltin() { none() } - - override ControlFlowNode getOrigin() { this = TClassMethod(result, _) } - - override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } - - override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } - - override int intValue() { none() } - - override string strValue() { none() } - - override predicate calleeAndOffset(Function scope, int paramOffset) { none() } - - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } - - override predicate attributesUnknown() { none() } - - override boolean isDescriptor() { result = true } - - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { - any(ObjectInternal obj).binds(instance, _, this) and - exists(ObjectInternal cls | - instance.isClass() = false and cls = instance.getClass() - or - instance.isClass() = true and cls = instance - | - value = TBoundMethod(cls, this.getFunction()) and - origin = CfgOrigin::unknown() - ) - } - - override predicate binds(ObjectInternal cls, string name, ObjectInternal descriptor) { - descriptor = this.getFunction() and - exists(ObjectInternal instance | - any(ObjectInternal obj).binds(instance, name, this) | - instance.isClass() = false and cls = instance.getClass() - or - instance.isClass() = true and cls = instance - ) - } - - override int length() { none() } - -} - -class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { - - override string toString() { - result = "staticmethod()" - } - - override boolean booleanValue() { result = true } - - override predicate introduced(ControlFlowNode node, PointsToContext context) { - exists(CallableObjectInternal function | - this = TStaticMethod(node, function) and - static_method(node, function, context) - ) - } - - CallableObjectInternal getFunction() { - this = TStaticMethod(_, result) - } - - override ClassDecl getClassDeclaration() { none() } - - override boolean isClass() { result = false } - - override ObjectInternal getClass() { result = ObjectInternal::builtin("staticmethod") } - - override boolean isComparable() { none() } - - override Builtin getBuiltin() { none() } - - override ControlFlowNode getOrigin() { this = TStaticMethod(result, _) } - - override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } - - override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } - - override int intValue() { none() } - - override string strValue() { none() } - - override predicate calleeAndOffset(Function scope, int paramOffset) { - this.getFunction().calleeAndOffset(scope, paramOffset) - } - - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } - - override predicate attributesUnknown() { none() } - - override boolean isDescriptor() { result = true } - - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { - any(ObjectInternal obj).binds(instance, _, this) and - value = this.getFunction() and origin = CfgOrigin::unknown() - } - - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } - - override int length() { none() } - -} diff --git a/python/ql/src/semmle/python/objects/Descriptors.qll b/python/ql/src/semmle/python/objects/Descriptors.qll new file mode 100644 index 000000000000..89dea82d6156 --- /dev/null +++ b/python/ql/src/semmle/python/objects/Descriptors.qll @@ -0,0 +1,218 @@ +import python + +private import semmle.python.objects.TObject +private import semmle.python.objects.ObjectInternal +private import semmle.python.pointsto.PointsTo +private import semmle.python.pointsto.PointsToContext +private import semmle.python.pointsto.MRO +private import semmle.python.types.Builtins + +class PropertyInternal extends ObjectInternal, TProperty { + + override string toString() { + result = "property" + this.getName() + } + + override boolean booleanValue() { result = true } + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + this = TProperty(node, context, _) + } + + override ClassDecl getClassDeclaration() { none() } + + override boolean isClass() { result = false } + + override ObjectInternal getClass() { result = ObjectInternal::property() } + + CallableObjectInternal getGetter() { + this = TProperty(_, _, result) + } + + override boolean isComparable() { result = true } + + override Builtin getBuiltin() { none() } + + string getName() { + result = this.getGetter().getName() + } + + override ControlFlowNode getOrigin() { this = TProperty(result, _, _) } + + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } + + override int intValue() { none() } + + override string strValue() { none() } + + override predicate calleeAndOffset(Function scope, int paramOffset) { none() } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate attributesUnknown() { none() } + + override boolean isDescriptor() { result = true } + + override int length() { none() } + + private Context getContext() { this = TProperty(_,result, _) } + + CallableObjectInternal getSetter() { + exists(CallNode call, AttrNode setter | + call.getFunction() = setter and + PointsToInternal::pointsTo(setter.getObject("setter"), this.getContext(), this, _) and + PointsToInternal::pointsTo(call.getArg(0), this.getContext(), result, _) + ) + } + + override predicate binds(ObjectInternal cls, string name, ObjectInternal descriptor) { none() } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { + /* Just give an unknown value for now. We could improve this, but it would mean + * changing Contexts to account for property accesses. + */ + exists(AttrNode attr, ClassObjectInternal cls, string name | + name = this.getName() and + PointsToInternal::pointsTo(attr.getObject(name), _, instance, _) and + instance.getClass() = cls and + cls.lookup(name, this, _) and + origin = CfgOrigin::fromCfgNode(attr) and value = ObjectInternal::unknown() + ) + } + +} + +class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { + + override string toString() { + result = "classmethod(" + this.getFunction() + ")" + } + + override boolean booleanValue() { result = true } + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + exists(CallableObjectInternal function | + this = TClassMethod(node, function) and + class_method(node, function, context) + ) + } + + CallableObjectInternal getFunction() { + this = TClassMethod(_, result) + } + + override ClassDecl getClassDeclaration() { none() } + + override boolean isClass() { result = false } + + override ObjectInternal getClass() { result = ObjectInternal::classMethod() } + + override boolean isComparable() { result = true } + + override Builtin getBuiltin() { none() } + + override ControlFlowNode getOrigin() { this = TClassMethod(result, _) } + + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } + + override int intValue() { none() } + + override string strValue() { none() } + + override predicate calleeAndOffset(Function scope, int paramOffset) { none() } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate attributesUnknown() { none() } + + override boolean isDescriptor() { result = true } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { + any(ObjectInternal obj).binds(instance, _, this) and + exists(ObjectInternal cls | + instance.isClass() = false and cls = instance.getClass() + or + instance.isClass() = true and cls = instance + | + value = TBoundMethod(cls, this.getFunction()) and + origin = CfgOrigin::unknown() + ) + } + + override predicate binds(ObjectInternal cls, string name, ObjectInternal descriptor) { + descriptor = this.getFunction() and + exists(ObjectInternal instance | + any(ObjectInternal obj).binds(instance, name, this) | + instance.isClass() = false and cls = instance.getClass() + or + instance.isClass() = true and cls = instance + ) + } + + override int length() { none() } + +} + +class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { + + override string toString() { + result = "staticmethod()" + } + + override boolean booleanValue() { result = true } + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + exists(CallableObjectInternal function | + this = TStaticMethod(node, function) and + static_method(node, function, context) + ) + } + + CallableObjectInternal getFunction() { + this = TStaticMethod(_, result) + } + + override ClassDecl getClassDeclaration() { none() } + + override boolean isClass() { result = false } + + override ObjectInternal getClass() { result = ObjectInternal::builtin("staticmethod") } + + override boolean isComparable() { result = true } + + override Builtin getBuiltin() { none() } + + override ControlFlowNode getOrigin() { this = TStaticMethod(result, _) } + + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } + + override int intValue() { none() } + + override string strValue() { none() } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + this.getFunction().calleeAndOffset(scope, paramOffset) + } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate attributesUnknown() { none() } + + override boolean isDescriptor() { result = true } + + override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { + any(ObjectInternal obj).binds(instance, _, this) and + value = this.getFunction() and origin = CfgOrigin::unknown() + } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + + override int length() { none() } + +} diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 5c24570bfd95..f0fb94aa43ea 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -352,7 +352,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal { result = ObjectInternal::builtin("super") } - override boolean isComparable() { none() } + override boolean isComparable() { result = false } override Builtin getBuiltin() { none() } diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 10b8f5f6e5fe..c6c55c712712 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -11,6 +11,7 @@ import semmle.python.objects.Instances import semmle.python.objects.Callables import semmle.python.objects.Constants import semmle.python.objects.Sequences +import semmle.python.objects.Descriptors class ObjectInternal extends TObject { @@ -396,6 +397,9 @@ module ObjectInternal { result = TBuiltinClassObject(Builtin::special("NoneType")) } + ObjectInternal property() { + result = TBuiltinClassObject(Builtin::special("property")) + } } /** Helper for boolean predicates returning both `true` and `false` */ diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index f937eb03e700..2abadaee539a 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -125,6 +125,11 @@ newtype TObject = } or TType() + or + TProperty(CallNode call, Context ctx, CallableObjectInternal getter) { + PointsToInternal::pointsTo(call.getFunction(), ctx, ObjectInternal::property(), _) and + PointsToInternal::pointsTo(call.getArg(0), ctx, getter, _) + } private predicate is_power_2(int n) { n = 1 or From 46b9ef79b42ae1305768b1fd96afca53abfbff39 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 12 Apr 2019 15:42:56 +0100 Subject: [PATCH 049/128] Python points-to: Fix up neverReturns() and return value of __import__(). --- .../src/semmle/python/objects/Callables.qll | 19 ++++++++++++++++++- .../src/semmle/python/objects/ObjectAPI.qll | 3 +-- .../src/semmle/python/pointsto/PointsTo.qll | 17 +++++++++++++++++ .../semmle/python/security/TaintTracking.qll | 2 +- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index edcace21ede2..f438a40f88d1 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -52,6 +52,7 @@ abstract class CallableObjectInternal extends ObjectInternal { override int length() { none() } + abstract predicate neverReturns(); } @@ -146,6 +147,10 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti result.getNode() = this.getScope().getArgByName(name) } + + override predicate neverReturns() { + InterProceduralPointsTo::neverReturns(this.getScope()) + } } @@ -220,9 +225,11 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc or func = Builtin::builtin("intern") and result = Builtin::special("str") or + func = Builtin::builtin("__import__") and result = Builtin::special("ModuleType") + or /* Fix a few minor inaccuracies in the CPython analysis */ ext_rettype(func, result) and not ( - func = Builtin::builtin("__import__") and result = Builtin::special("NoneType") + func = Builtin::builtin("__import__") or func = Builtin::builtin("compile") and result = Builtin::special("NoneType") or @@ -251,6 +258,10 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc none() } + override predicate neverReturns() { + this = Module::named("sys").attr("exit") + } + } @@ -334,6 +345,8 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod none() } + override predicate neverReturns() { none() } + } class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { @@ -405,6 +418,10 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { result = this.getFunction().getParameterByName(name) } + override predicate neverReturns() { + this.getFunction().neverReturns() + } + } diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 6eb8f97f4f0f..cf724077ce33 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -103,8 +103,7 @@ class CallableValue extends Value { } predicate neverReturns() { - // TO DO.. - none() + this.(CallableObjectInternal).neverReturns() } Function getScope() { diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 91cdf36f814e..7599dac78bbf 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1022,6 +1022,23 @@ module InterProceduralPointsTo { BaseFlow::reaches_exit(evar) } + /** INTERNAL -- Use `FunctionObject.neverReturns()` instead. + * Whether function `func` never returns. Slightly conservative approximation, this predicate may be false + * for a function that can never return. */ + cached predicate neverReturns(Function f) { + /* A Python function never returns if it has no normal exits that are not dominated by a + * call to a function which itself never returns. + */ + forall(BasicBlock exit | + exit = f.getANormalExit().getBasicBlock() | + exists(FunctionObject callee, BasicBlock call | + callee.getACall().getBasicBlock() = call and + callee.neverReturns() and + call.dominates(exit) + ) + ) + } + } /** Gets the `value, origin` that `f` would refer to if it has not been assigned some other value */ diff --git a/python/ql/src/semmle/python/security/TaintTracking.qll b/python/ql/src/semmle/python/security/TaintTracking.qll index 3cae6ea5e2a6..b97b0d0861a3 100755 --- a/python/ql/src/semmle/python/security/TaintTracking.qll +++ b/python/ql/src/semmle/python/security/TaintTracking.qll @@ -1268,7 +1268,7 @@ library module TaintFlowImplementation { | test.getSense() = true and not exists(kind.getClass()) or - test.getSense() = true and kind.getClass().getASuperType() = cls + test.getSense() = true and kind.getType().getASuperType() = cls or test.getSense() = false and not kind.getType().getASuperType() = cls ) From 90bbfd3b163d298eda2969cdacc50d788f79de3d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 12 Apr 2019 16:14:14 +0100 Subject: [PATCH 050/128] Python: Add library tests for django. --- .../library-tests/web/django/Sinks.expected | 6 +++ .../ql/test/library-tests/web/django/Sinks.ql | 13 ++++++ .../library-tests/web/django/Sources.expected | 2 + .../test/library-tests/web/django/Sources.ql | 10 +++++ .../ql/test/library-tests/web/django/options | 2 + .../ql/test/library-tests/web/django/test.py | 40 +++++++++++++++++++ .../Security/lib/django/http/__init__.py | 2 + .../Security/lib/django/http/request.py | 2 + .../Security/lib/django/http/response.py | 2 + 9 files changed, 79 insertions(+) create mode 100644 python/ql/test/library-tests/web/django/Sinks.expected create mode 100644 python/ql/test/library-tests/web/django/Sinks.ql create mode 100644 python/ql/test/library-tests/web/django/Sources.expected create mode 100644 python/ql/test/library-tests/web/django/Sources.ql create mode 100644 python/ql/test/library-tests/web/django/options create mode 100644 python/ql/test/library-tests/web/django/test.py create mode 100644 python/ql/test/query-tests/Security/lib/django/http/__init__.py create mode 100644 python/ql/test/query-tests/Security/lib/django/http/request.py create mode 100644 python/ql/test/query-tests/Security/lib/django/http/response.py diff --git a/python/ql/test/library-tests/web/django/Sinks.expected b/python/ql/test/library-tests/web/django/Sinks.expected new file mode 100644 index 000000000000..82d1b71e53a6 --- /dev/null +++ b/python/ql/test/library-tests/web/django/Sinks.expected @@ -0,0 +1,6 @@ +| test.py:18 | Str | externally controlled string | +| test.py:21 | BinaryExpr | externally controlled string | +| test.py:24 | BinaryExpr | externally controlled string | +| test.py:25 | BinaryExpr | externally controlled string | +| test.py:26 | BinaryExpr | externally controlled string | +| test.py:34 | BinaryExpr | externally controlled string | diff --git a/python/ql/test/library-tests/web/django/Sinks.ql b/python/ql/test/library-tests/web/django/Sinks.ql new file mode 100644 index 000000000000..5bdf37e4f44d --- /dev/null +++ b/python/ql/test/library-tests/web/django/Sinks.ql @@ -0,0 +1,13 @@ + +import python + +import semmle.python.web.HttpRequest +import semmle.python.web.HttpResponse +import semmle.python.web.django.Db +import semmle.python.web.django.Model + +import semmle.python.security.strings.Untrusted + +from TaintSink sink, TaintKind kind +where sink.sinks(kind) +select sink.getLocation().toString(), sink.(ControlFlowNode).getNode().toString(), kind diff --git a/python/ql/test/library-tests/web/django/Sources.expected b/python/ql/test/library-tests/web/django/Sources.expected new file mode 100644 index 000000000000..8f421a2d1694 --- /dev/null +++ b/python/ql/test/library-tests/web/django/Sources.expected @@ -0,0 +1,2 @@ +| test.py:11 | request | django.request.HttpRequest | +| test.py:31 | request | django.request.HttpRequest | diff --git a/python/ql/test/library-tests/web/django/Sources.ql b/python/ql/test/library-tests/web/django/Sources.ql new file mode 100644 index 000000000000..aece91f44e21 --- /dev/null +++ b/python/ql/test/library-tests/web/django/Sources.ql @@ -0,0 +1,10 @@ + +import python + +import semmle.python.web.HttpRequest +import semmle.python.security.strings.Untrusted + + +from TaintSource src, TaintKind kind +where src.isSourceOf(kind) +select src.getLocation().toString(), src.(ControlFlowNode).getNode().toString(), kind diff --git a/python/ql/test/library-tests/web/django/options b/python/ql/test/library-tests/web/django/options new file mode 100644 index 000000000000..3eb8fa37213e --- /dev/null +++ b/python/ql/test/library-tests/web/django/options @@ -0,0 +1,2 @@ +semmle-extractor-options: --max-import-depth=3 --lang=3 -p ../../../query-tests/Security/lib/ +optimize: true diff --git a/python/ql/test/library-tests/web/django/test.py b/python/ql/test/library-tests/web/django/test.py new file mode 100644 index 000000000000..e6c39c30a32d --- /dev/null +++ b/python/ql/test/library-tests/web/django/test.py @@ -0,0 +1,40 @@ + +from django.conf.urls import patterns, url +from django.db import connection, models +from django.db.models.expressions import RawSQL +from django.http.response import HttpResponse +import base64 + +class Name(models.Model): + pass + +def save_name(request): + + if request.method == 'POST': + name = request.POST.get('name') + curs = connection.cursor() + #GOOD -- Using parameters + curs.execute( + "insert into names_file ('name') values ('%s')", name) + #BAD -- Using string formatting + curs.execute( + "insert into names_file ('name') values ('%s')" % name) + + #BAD -- other ways of executing raw SQL code with string interpolation + Name.objects.annotate(RawSQL("insert into names_file ('name') values ('%s')" % name)) + Name.objects.raw("insert into names_file ('name') values ('%s')" % name) + Name.objects.extra("insert into names_file ('name') values ('%s')" % name) + +urlpatterns1 = patterns(url(r'^save_name/$', + save_name, name='save_name')) + +def maybe_xss(request): + first_name = request.POST.get('first_name', '') + resp = HttpResponse() + resp.write("first name is " + first_name) + return resp + +urlpatterns2 = [ + # Route to code_execution + url(r'^maybe_xss$', maybe_xss, name='maybe_xss') +] diff --git a/python/ql/test/query-tests/Security/lib/django/http/__init__.py b/python/ql/test/query-tests/Security/lib/django/http/__init__.py new file mode 100644 index 000000000000..962077dbad6f --- /dev/null +++ b/python/ql/test/query-tests/Security/lib/django/http/__init__.py @@ -0,0 +1,2 @@ +from .response import HttpResponse +from .request import HttpRequest diff --git a/python/ql/test/query-tests/Security/lib/django/http/request.py b/python/ql/test/query-tests/Security/lib/django/http/request.py new file mode 100644 index 000000000000..93afdaf29b76 --- /dev/null +++ b/python/ql/test/query-tests/Security/lib/django/http/request.py @@ -0,0 +1,2 @@ +class HttpRequest(object): + pass diff --git a/python/ql/test/query-tests/Security/lib/django/http/response.py b/python/ql/test/query-tests/Security/lib/django/http/response.py new file mode 100644 index 000000000000..ae101562df47 --- /dev/null +++ b/python/ql/test/query-tests/Security/lib/django/http/response.py @@ -0,0 +1,2 @@ +class HttpResponse(object): + pass From 9d40a6cd8c255c94d0d205edee0c03c401ac8182 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 15 Apr 2019 10:21:26 +0100 Subject: [PATCH 051/128] Python points-to: restore getArgumentForCall() API method. --- .../src/semmle/python/objects/Callables.qll | 18 ++++++++++ .../ql/src/semmle/python/objects/Classes.qll | 4 +++ .../src/semmle/python/objects/ObjectAPI.qll | 36 ++++++++++++++----- .../semmle/python/objects/ObjectInternal.qll | 2 ++ .../semmle/python/types/FunctionObject.qll | 4 +-- 5 files changed, 53 insertions(+), 11 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index f438a40f88d1..f56b0676bd3c 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -53,6 +53,7 @@ abstract class CallableObjectInternal extends ObjectInternal { override int length() { none() } abstract predicate neverReturns(); + } @@ -151,6 +152,11 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti override predicate neverReturns() { InterProceduralPointsTo::neverReturns(this.getScope()) } + + override predicate functionAndOffset(CallableObjectInternal function, int offset) { + function = this and offset = 0 + } + } @@ -262,6 +268,10 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc this = Module::named("sys").attr("exit") } + override predicate functionAndOffset(CallableObjectInternal function, int offset) { + function = this and offset = 0 + } + } @@ -347,6 +357,10 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod override predicate neverReturns() { none() } + override predicate functionAndOffset(CallableObjectInternal function, int offset) { + function = this and offset = 0 + } + } class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { @@ -422,6 +436,10 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { this.getFunction().neverReturns() } + override predicate functionAndOffset(CallableObjectInternal function, int offset) { + function = this.getFunction() and offset = 1 + } + } diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index b9f99ce37a13..e2849e49dfbb 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -122,6 +122,10 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject override boolean isComparable() { result = true } + override predicate functionAndOffset(CallableObjectInternal function, int offset) { + this.lookup("__init__", function, _) and offset = 1 + } + } class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObject { diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index cf724077ce33..8d4127b04c7e 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -118,17 +118,35 @@ class CallableValue extends Value { result = this.(CallableObjectInternal).getParameterByName(name) } - ControlFlowNode getArgumentForCall(CallNode call, NameNode param) { - this.getACall() = call and - ( - exists(int n | call.getArg(n) = result and param = this.getParameter(n)) + ControlFlowNode getArgumentForCall(CallNode call, int n) { + exists(ObjectInternal called, int offset | + PointsToInternal::pointsTo(call.getFunction(), _, called, _) and + called.functionAndOffset(this, offset) + | + call.getArg(n-offset) = result or - exists(string name | call.getArgByName(name) = result and param = this.getParameterByName(name)) + exists(string name | call.getArgByName(name) = result and this.(PythonFunctionObjectInternal).getScope().getArg(n+offset).getName() = name) + or + called instanceof BoundMethodObjectInternal and + offset = 1 and n = 0 and result = call.getFunction().(AttrNode).getObject() ) - or - exists(BoundMethodObjectInternal bm | - result = getArgumentForCall(call, param) and - this = bm.getFunction() + } + + ControlFlowNode getNamedArgumentForCall(CallNode call, string name) { + exists(CallableObjectInternal called, int offset | + PointsToInternal::pointsTo(call.getFunction(), _, called, _) and + called.functionAndOffset(this, offset) + | + exists(int n | + call.getArg(n) = result and + this.(PythonFunctionObjectInternal).getScope().getArg(n+offset).getName() = name + ) + or + call.getArgByName(name) = result and + exists(this.(PythonFunctionObjectInternal).getScope().getArgByName(name)) + or + called instanceof BoundMethodObjectInternal and + offset = 1 and name = "self" and result = call.getFunction().(AttrNode).getObject() ) } diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index c6c55c712712..433e24db386a 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -103,6 +103,8 @@ class ObjectInternal extends TObject { */ abstract int length(); + predicate functionAndOffset(CallableObjectInternal function, int offset) { none() } + } diff --git a/python/ql/src/semmle/python/types/FunctionObject.qll b/python/ql/src/semmle/python/types/FunctionObject.qll index 37f7f506422d..55a30bd673d9 100644 --- a/python/ql/src/semmle/python/types/FunctionObject.qll +++ b/python/ql/src/semmle/python/types/FunctionObject.qll @@ -71,14 +71,14 @@ abstract class FunctionObject extends Object { This predicate will correctly handle `x.y()`, treating `x` as the zeroth argument. */ ControlFlowNode getArgumentForCall(CallNode call, int n) { - result = theCallable().getArgumentForCall(call, this.getFunction().getArg(n).asName().getAFlowNode()) + result = theCallable().getArgumentForCall(call, n) } /** Gets the `ControlFlowNode` that will be passed as the named argument to `this` when called at `call`. This predicate will correctly handle `x.y()`, treating `x` as the self argument. */ ControlFlowNode getNamedArgumentForCall(CallNode call, string name) { - result = theCallable().getArgumentForCall(call, this.getFunction().getArgByName(name).asName().getAFlowNode()) + result = theCallable().getNamedArgumentForCall(call, name) } /** Whether this function never returns. This is an approximation. From 06ab6716293d83cf76dd4db4db39da432d72e9ee Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 15 Apr 2019 12:50:51 +0100 Subject: [PATCH 052/128] Python points-to. Fix descriptor behviour of Python functions. --- python/ql/src/semmle/python/objects/Callables.qll | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index f56b0676bd3c..4710ce91810a 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -129,7 +129,12 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti override boolean isDescriptor() { result = true } override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { + instance.isClass() = false and value = TBoundMethod(instance, this) and origin = CfgOrigin::unknown() + or + any(ObjectInternal obj).binds(instance, _, this) and + instance.isClass() = true and + value = this and origin = CfgOrigin::fromCfgNode(this.getOrigin()) } override CallNode getACall(PointsToContext ctx) { From 2e6c3c9ee34fd749408f027b0bdb67237ceddfab Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 15 Apr 2019 12:52:37 +0100 Subject: [PATCH 053/128] Python points-to: Support classes created dynamically as instances of meta-class. --- .../ql/src/semmle/python/objects/Classes.qll | 50 +++++++++++++++++++ .../ql/src/semmle/python/objects/TObject.qll | 5 ++ .../src/semmle/python/pointsto/PointsTo.qll | 2 +- .../src/semmle/python/types/ClassObject.qll | 2 +- 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index e2849e49dfbb..a5e93d940788 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -277,3 +277,53 @@ class TypeInternal extends ClassObjectInternal, TType { override predicate attributesUnknown() { any() } } + +class DynamicallyCreatedClass extends ClassObjectInternal, TDynamicClass { + + override string toString() { + result = this.getOrigin().getNode().toString() + } + + override ObjectInternal getClass() { + this = TDynamicClass(_, result, _) + } + + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + none() + } + + override predicate lookup(string name, ObjectInternal value, CfgOrigin origin) { + exists(ClassObjectInternal decl | + decl = Types::getMro(this).findDeclaringClass(name) | + Types::declaredAttribute(decl, name, value, origin) + ) + } + + override Builtin getBuiltin() { + none() + } + + override ControlFlowNode getOrigin() { + this = TDynamicClass(result, _, _) + } + + override predicate attributesUnknown() { any() } + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + this = TDynamicClass(node, _, context) + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + + override boolean isComparable() { result = true } + + override ClassDecl getClassDeclaration() { none() } + +} + diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 2abadaee539a..cce3fb6c679e 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -130,6 +130,11 @@ newtype TObject = PointsToInternal::pointsTo(call.getFunction(), ctx, ObjectInternal::property(), _) and PointsToInternal::pointsTo(call.getArg(0), ctx, getter, _) } + or + TDynamicClass(ControlFlowNode instantiation, ClassObjectInternal metacls, PointsToContext context) { + PointsToInternal::pointsTo(instantiation.(CallNode).getFunction(), context, metacls, _) and + Types::getMro(metacls).contains(TType()) + } private predicate is_power_2(int n) { n = 1 or diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 7599dac78bbf..3fe20c8fb803 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1445,7 +1445,7 @@ cached module Types { ) } - cached ClassObjectInternal getBase(ClassObjectInternal cls, int n) { + cached ObjectInternal getBase(ClassObjectInternal cls, int n) { result.getBuiltin() = cls.getBuiltin().getBaseClass() and n = 0 or exists(Class pycls | diff --git a/python/ql/src/semmle/python/types/ClassObject.qll b/python/ql/src/semmle/python/types/ClassObject.qll index acc7582e3d16..0e089a2f4935 100644 --- a/python/ql/src/semmle/python/types/ClassObject.qll +++ b/python/ql/src/semmle/python/types/ClassObject.qll @@ -68,7 +68,7 @@ class ClassObject extends Object { /** Gets a super class of this class (includes transitive super classes) or this class */ ClassObject getAnImproperSuperType() { - result = Types::getMro(theClass()).getAnItem().getSource() + result = this.getABaseType*() } /** Whether this class is a new style class. From cd34e23c4c124c2352dd356c0461dc8d745f1a52 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 15 Apr 2019 18:14:32 +0100 Subject: [PATCH 054/128] Python points-to: Bring attribute points-to to effective parity with old points-to. --- .../src/semmle/python/objects/Constants.qll | 2 +- .../src/semmle/python/objects/Instances.qll | 12 ++ .../semmle/python/objects/ObjectInternal.qll | 18 +-- .../ql/src/semmle/python/objects/TObject.qll | 5 +- .../src/semmle/python/pointsto/PointsTo.qll | 125 ++++++++++++++---- 5 files changed, 120 insertions(+), 42 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index b297dc995b62..25e05efaabc2 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -198,7 +198,7 @@ class FloatObjectInternal extends ConstantObjectInternal, TFloat { override string toString() { if this.floatValue() = this.floatValue().floor() then ( - result = "float " + this.floatValue().toString() + ".0" + result = "float " + this.floatValue().floor().toString() + ".0" ) else ( result = "float " + this.floatValue().toString() ) diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index f0fb94aa43ea..b22e9bea7b1b 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -79,6 +79,18 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { PointsToInternal::attributeRequired(this, name) and instance_getattr(this, Types::getMro(this.getClass()), name, value, origin) + or + exists(EssaVariable self, PythonFunctionObjectInternal init, Context callee | + BaseFlow::reaches_exit(self) and + self.getSourceVariable().(Variable).isSelf() and + self.getScope() = init.getScope() and + exists(CallNode call, Context caller, ClassObjectInternal cls | + this = TSpecificInstance(call, cls, caller) and + callee.fromCall(this.getOrigin(), caller) and + cls.lookup("__init__", init, _) + ) and + AttributePointsTo::variableAttributePointsTo(self, callee, name, value, origin) + ) } override predicate attributesUnknown() { any() } diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 433e24db386a..9eb2cc4f8d26 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -366,21 +366,9 @@ module ObjectInternal { } ObjectInternal fromBuiltin(Builtin b) { - result = TInt(b.intValue()) - or - result = TString(b.strValue()) - or - result = TBuiltinClassObject(b) - or - result = TBuiltinFunctionObject(b) - or - result = TBuiltinOpaqueObject(b) - or - result = TBuiltinModuleObject(b) - or - result = TBuiltinMethodObject(b) - or - result = TBuiltinTuple(b) + b = result.getBuiltin() and + not b = Builtin::unknown() and + not b = Builtin::unknownType() } ObjectInternal classMethod() { diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index cce3fb6c679e..cbe3a963b649 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -131,8 +131,9 @@ newtype TObject = PointsToInternal::pointsTo(call.getArg(0), ctx, getter, _) } or - TDynamicClass(ControlFlowNode instantiation, ClassObjectInternal metacls, PointsToContext context) { - PointsToInternal::pointsTo(instantiation.(CallNode).getFunction(), context, metacls, _) and + TDynamicClass(CallNode instantiation, ClassObjectInternal metacls, PointsToContext context) { + PointsToInternal::pointsTo(instantiation.getFunction(), context, metacls, _) and + not count(instantiation.getAnArg()) = 1 and Types::getMro(metacls).contains(TType()) } diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 3fe20c8fb803..c2485a79afca 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -126,7 +126,7 @@ module PointsTo { deprecated predicate points_to(ControlFlowNode f, PointsToContext context, Object obj, ClassObject cls, ControlFlowNode origin) { exists(Value value | - pointsToValue(f, context, value, origin) and + PointsToInternal::pointsTo(f, context, value, origin) and cls = value.getClass().getSource() | obj = value.getSource() or not exists(value.getSource()) and obj = origin @@ -148,14 +148,6 @@ module PointsTo { ) } - private predicate pointsToValue(ControlFlowNode f, PointsToContext context, Value value, ControlFlowNode origin) { - PointsToInternal::pointsTo(f, context, value, origin) - or - exists(string name | - AttributePointsTo::attributePointsTo(f.(AttrNode).getObject(name), context, name, value, origin) - ) - } - deprecated predicate ssa_variable_points_to(EssaVariable var, PointsToContext context, Object obj, ClassObject cls, CfgOrigin origin) { exists(Value value | @@ -199,6 +191,8 @@ cached module PointsToInternal { InterModulePointsTo::from_import_points_to(f, context, value, origin) or InterProceduralPointsTo::call_points_to(f, context, value, origin) + or + AttributePointsTo::pointsTo(f, context, value, origin) // To do... More stuff here :) // or // f.(CustomPointsToFact).pointsTo(context, value, origin) @@ -1135,8 +1129,10 @@ module Expressions { private boolean otherComparisonEvaluatesTo(CompareNode comp, PointsToContext context, ControlFlowNode operand, ObjectInternal opvalue) { exists(Cmpop op | - comp.operands(operand, op, _) and - (op instanceof In or op instanceof NotIn) | + comp.operands(operand, op, _) or + comp.operands(_, op, operand) + | + (op instanceof In or op instanceof NotIn) and PointsToInternal::pointsTo(operand, context, opvalue, _) ) and result = maybe() } @@ -1694,19 +1690,100 @@ cached module Types { module AttributePointsTo { - predicate attributePointsTo(ControlFlowNode f, Context context, string name, ObjectInternal value, ControlFlowNode origin) { - exists(ObjectInternal defobj, Context prev, AttributeAssignment def, ObjectInternal useobj | - PointsToInternal::pointsTo(f, context, useobj, _) and - PointsToInternal::variablePointsTo(def.getInput(), prev, defobj, _) and - PointsToInternal::pointsTo(def.getValue(), prev, value, origin) and name = def.getName() - | - prev.getOuter*().getCall().getBasicBlock().reaches(context.getOuter*().getCall().getBasicBlock()) and - useobj.(SelfInstanceInternal).getClass() = defobj.(SelfInstanceInternal).getClass() - or - def.getScope().getScope*().precedes(f.getScope().getScope*()) and - useobj.(SelfInstanceInternal).getClass() = defobj.(SelfInstanceInternal).getClass() - or - def.getDefiningNode().getBasicBlock().dominates(f.getBasicBlock()) and defobj = useobj + predicate pointsTo(AttrNode f, Context context, ObjectInternal value, ControlFlowNode origin) { + exists(EssaVariable var, string name, CfgOrigin orig | + var.getASourceUse() = f.getObject(name) and + variableAttributePointsTo(var, context, name, value, orig) and + origin = orig.asCfgNodeOrHere(f) + ) + } + + predicate variableAttributePointsTo(EssaVariable var, Context context, string name, ObjectInternal value, CfgOrigin origin) { + definitionAttributePointsTo(var.getDefinition(), context, name, value, origin) + or + exists(EssaVariable prev | + var.getDefinition().(PhiFunction).getShortCircuitInput() = prev and + variableAttributePointsTo(prev, context, name, value, origin) + ) + } + + predicate definitionAttributePointsTo(EssaDefinition def, Context context, string name, ObjectInternal value, CfgOrigin origin) { + variableAttributePointsTo(def.(PhiFunction).getAnInput(), context, name, value, origin) + or + piNodeAttributePointsTo(def, context, name, value, origin) + or + refinementAttributePointsTo(def, context, name, value, origin) + or + selfParameterAttributePointsTo(def, context, name, value, origin) + or + selfMethodCallsitePointsTo(def, context, name, value, origin) + } + + pragma [noinline] + private predicate refinementAttributePointsTo(EssaNodeRefinement def, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { + attributeAssignmentAttributePointsTo(def, context, name, value, origin) + or + attributeDeleteAttributePointsTo(def, context, name, value, origin) + or + uniEdgedPhiAttributePointsTo(def, context, name, value, origin) + } + + + /** Attribute deletions have no effect as far as value tracking is concerned. */ + pragma [noinline] + private predicate attributeAssignmentAttributePointsTo(AttributeAssignment def, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { + def.getName() != name and + variableAttributePointsTo(def.getInput(), context, name, value, origin) + or + def.getName() = name and + exists(ControlFlowNode cfgnode | + PointsToInternal::pointsTo(def.getValue(), context, value, cfgnode) and + origin = CfgOrigin::fromCfgNode(cfgnode) + ) + } + + /** Attribute deletions have no effect as far as value tracking is concerned. */ + pragma [noinline] + private predicate attributeDeleteAttributePointsTo(EssaAttributeDeletion def, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { + def.getName() != name and + variableAttributePointsTo(def.getInput(), context, name, value, origin) + } + + private predicate uniEdgedPhiAttributePointsTo(SingleSuccessorGuard uniphi, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { + variableAttributePointsTo(uniphi.getInput(), context, name, value, origin) + } + + private predicate piNodeAttributePointsTo(PyEdgeRefinement pi, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { + variableAttributePointsTo(pi.getInput(), context, name, value, origin) + } + + private predicate selfParameterAttributePointsTo(ParameterDefinition def, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { + exists(MethodCallsiteRefinement call, Function func, PointsToContext caller | + selfMethodCall(call, caller, func, context) and + def.isSelf() and def.getScope() = func and + variableAttributePointsTo(call.getInput(), caller, name, value, origin) + ) + } + + /** Pass through for `self` for the implicit re-definition of `self` in `self.foo()`. */ + private predicate selfMethodCallsitePointsTo(MethodCallsiteRefinement def, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { + /* The value of self remains the same, only the attributes may change */ + exists(Function func, PointsToContext callee, EssaVariable exit_self | + selfMethodCall(def, context, func, callee) and + exit_self.getSourceVariable().(Variable).isSelf() and + exit_self.getScope() = func and + BaseFlow::reaches_exit(exit_self) and + variableAttributePointsTo(exit_self, context, name, value, origin) + ) + } + + private predicate selfMethodCall(MethodCallsiteRefinement def, PointsToContext caller, Function func, PointsToContext callee) { + def.getInput().getSourceVariable().(Variable).isSelf() and + exists(PythonFunctionObjectInternal method, CallNode call | + method.getScope() = func and + call = method.getACall() and + call = def.getDefiningNode() and + callee.fromCall(call, caller) ) } From dffbf698d24fd76dd3285c7c62974b0d66179c07 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Apr 2019 17:30:14 +0100 Subject: [PATCH 055/128] Python points-to: improve performance. --- .../src/analysis/ContextMarginalEfficiency.ql | 2 +- python/ql/src/semmle/python/Flow.qll | 10 + .../src/semmle/python/objects/Callables.qll | 40 +-- .../ql/src/semmle/python/objects/Classes.qll | 25 +- .../src/semmle/python/objects/Constants.qll | 10 +- .../src/semmle/python/objects/Descriptors.qll | 56 ++-- .../src/semmle/python/objects/Instances.qll | 121 ++++---- .../ql/src/semmle/python/objects/Modules.qll | 25 +- .../semmle/python/objects/ObjectInternal.qll | 48 +++- .../src/semmle/python/objects/Sequences.qll | 10 +- .../ql/src/semmle/python/objects/TObject.qll | 17 +- python/ql/src/semmle/python/pointsto/MRO.qll | 7 + .../src/semmle/python/pointsto/PointsTo.qll | 266 +++++++++++------- .../python/pointsto/PointsToContext.qll | 3 + 14 files changed, 391 insertions(+), 249 deletions(-) diff --git a/python/ql/src/analysis/ContextMarginalEfficiency.ql b/python/ql/src/analysis/ContextMarginalEfficiency.ql index f48e05301233..99e659b4d196 100644 --- a/python/ql/src/analysis/ContextMarginalEfficiency.ql +++ b/python/ql/src/analysis/ContextMarginalEfficiency.ql @@ -29,4 +29,4 @@ total_size = strictcount(ControlFlowNode f, Object value, ClassObject cls, Point ) and efficiency = 100.0 * total_facts / total_size -select depth, total_facts, total_size, efficiency \ No newline at end of file +select depth, total_facts, total_size, efficiency diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll index fcec979f9070..36838818fe86 100755 --- a/python/ql/src/semmle/python/Flow.qll +++ b/python/ql/src/semmle/python/Flow.qll @@ -465,6 +465,16 @@ class CallNode extends ControlFlowNode { override Call getNode() { result = super.getNode() } + predicate isDecoratorCall() { + exists(FunctionExpr func | + this.getNode() = func.getADecoratorCall() + ) + or + exists(ClassExpr cls | + this.getNode() = cls.getADecoratorCall() + ) + } + } /** A control flow corresponding to an attribute expression, such as `value.attr` */ diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 4710ce91810a..59613e895b61 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -32,15 +32,15 @@ abstract class CallableObjectInternal extends ObjectInternal { abstract string getName(); - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } abstract Function getScope(); - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } abstract CallNode getACall(PointsToContext ctx); @@ -112,7 +112,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti } pragma [noinline] - override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { this.getScope().isProcedure() and obj = ObjectInternal::none_() and origin = CfgOrigin::fromCfgNode(this.getScope().getEntryNode()) @@ -128,15 +128,15 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti override boolean isDescriptor() { result = true } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { - instance.isClass() = false and - value = TBoundMethod(instance, this) and origin = CfgOrigin::unknown() - or - any(ObjectInternal obj).binds(instance, _, this) and - instance.isClass() = true and + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { + any(ObjectInternal obj).binds(cls, _, this) and value = this and origin = CfgOrigin::fromCfgNode(this.getOrigin()) } + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { + value = TBoundMethod(instance, this) and origin = CfgOrigin::unknown() + } + override CallNode getACall(PointsToContext ctx) { PointsTo::pointsTo(result.getFunction(), ctx, this, _) or @@ -255,7 +255,9 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc override boolean isDescriptor() { result = false } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } override CallNode getACall(PointsToContext ctx) { PointsTo::pointsTo(result.getFunction(), ctx, this, _) @@ -339,15 +341,15 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod override boolean isDescriptor() { result = true } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { - instance.isClass() = false and - value = TBoundMethod(instance, this) and origin = CfgOrigin::unknown() - or - any(ObjectInternal obj).binds(instance, _, this) and - instance.isClass() = true and + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { + any(ObjectInternal obj).binds(cls, _, this) and value = this and origin = CfgOrigin::unknown() } + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { + value = TBoundMethod(instance, this) and origin = CfgOrigin::unknown() + } + override CallNode getACall(PointsToContext ctx) { PointsTo::pointsTo(result.getFunction(), ctx, this, _) } @@ -423,7 +425,9 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { override boolean isDescriptor() { result = false } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } override CallNode getACall(PointsToContext ctx) { PointsTo::pointsTo(result.getFunction(), ctx, this, _) diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index a5e93d940788..111e1ca64e1c 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -35,9 +35,11 @@ abstract class ClassObjectInternal extends ObjectInternal { override boolean isDescriptor() { result = false } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { instance = this and PointsToInternal::attributeRequired(this, name) and this.lookup(name, descriptor, _) and @@ -47,14 +49,14 @@ abstract class ClassObjectInternal extends ObjectInternal { abstract predicate lookup(string name, ObjectInternal value, CfgOrigin origin); /** Approximation to descriptor protocol, skipping meta-descriptor protocol */ - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { exists(ObjectInternal descriptor, CfgOrigin desc_origin | this.lookup(name, descriptor, desc_origin) | descriptor.isDescriptor() = false and value = descriptor and origin = desc_origin or descriptor.isDescriptor() = true and - descriptor.descriptorGet(this, value, origin) + descriptor.descriptorGetClass(this, value, origin) ) } @@ -103,13 +105,10 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject } override predicate lookup(string name, ObjectInternal value, CfgOrigin origin) { - exists(ClassObjectInternal decl | - decl = Types::getMro(this).findDeclaringClass(name) | - Types::declaredAttribute(decl, name, value, origin) - ) + Types::getMro(this).lookup(name, value, origin) } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() @@ -166,7 +165,7 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec origin = CfgOrigin::unknown() } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() @@ -226,7 +225,7 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { none() } - override predicate attributesUnknown() { any() } + pragma [noinline] override predicate attributesUnknown() { any() } } @@ -274,7 +273,7 @@ class TypeInternal extends ClassObjectInternal, TType { none() } - override predicate attributesUnknown() { any() } + pragma [noinline] override predicate attributesUnknown() { any() } } @@ -311,7 +310,7 @@ class DynamicallyCreatedClass extends ClassObjectInternal, TDynamicClass { this = TDynamicClass(result, _, _) } - override predicate attributesUnknown() { any() } + pragma [noinline] override predicate attributesUnknown() { any() } override predicate introduced(ControlFlowNode node, PointsToContext context) { this = TDynamicClass(node, _, context) diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index 25e05efaabc2..723dcdf7f45e 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -37,17 +37,19 @@ abstract class ConstantObjectInternal extends ObjectInternal { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } override boolean isDescriptor() { result = false } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } } diff --git a/python/ql/src/semmle/python/objects/Descriptors.qll b/python/ql/src/semmle/python/objects/Descriptors.qll index 89dea82d6156..64cdf2030655 100644 --- a/python/ql/src/semmle/python/objects/Descriptors.qll +++ b/python/ql/src/semmle/python/objects/Descriptors.qll @@ -49,9 +49,9 @@ class PropertyInternal extends ObjectInternal, TProperty { override predicate calleeAndOffset(Function scope, int paramOffset) { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } override boolean isDescriptor() { result = true } @@ -67,18 +67,22 @@ class PropertyInternal extends ObjectInternal, TProperty { ) } - override predicate binds(ObjectInternal cls, string name, ObjectInternal descriptor) { none() } + pragma [noinline] override predicate binds(ObjectInternal cls, string name, ObjectInternal descriptor) { none() } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { + any(ObjectInternal obj).binds(cls, _, this) and + value = this and origin = CfgOrigin::fromCfgNode(this.getOrigin()) + } + + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { /* Just give an unknown value for now. We could improve this, but it would mean * changing Contexts to account for property accesses. */ - exists(AttrNode attr, ClassObjectInternal cls, string name | + exists(ClassObjectInternal cls, string name | name = this.getName() and - PointsToInternal::pointsTo(attr.getObject(name), _, instance, _) and - instance.getClass() = cls and + receiver_type(_, name, instance, cls) and cls.lookup(name, this, _) and - origin = CfgOrigin::fromCfgNode(attr) and value = ObjectInternal::unknown() + origin = CfgOrigin::unknown() and value = ObjectInternal::unknown() ) } @@ -125,25 +129,24 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { override predicate calleeAndOffset(Function scope, int paramOffset) { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } override boolean isDescriptor() { result = true } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { + value = TBoundMethod(cls, this.getFunction()) and + origin = CfgOrigin::unknown() + } + + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { any(ObjectInternal obj).binds(instance, _, this) and - exists(ObjectInternal cls | - instance.isClass() = false and cls = instance.getClass() - or - instance.isClass() = true and cls = instance - | - value = TBoundMethod(cls, this.getFunction()) and - origin = CfgOrigin::unknown() - ) + value = TBoundMethod(instance.getClass(), this.getFunction()) and + origin = CfgOrigin::unknown() } - override predicate binds(ObjectInternal cls, string name, ObjectInternal descriptor) { + pragma [noinline] override predicate binds(ObjectInternal cls, string name, ObjectInternal descriptor) { descriptor = this.getFunction() and exists(ObjectInternal instance | any(ObjectInternal obj).binds(instance, name, this) | @@ -200,18 +203,23 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { this.getFunction().calleeAndOffset(scope, paramOffset) } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } override boolean isDescriptor() { result = true } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { + any(ObjectInternal obj).binds(cls, _, this) and + value = this.getFunction() and origin = CfgOrigin::unknown() + } + + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { any(ObjectInternal obj).binds(instance, _, this) and value = this.getFunction() and origin = CfgOrigin::unknown() } - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } override int length() { none() } diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index b22e9bea7b1b..e3a851bf1e25 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -76,9 +76,16 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { none() } + pragma [nomagic] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { PointsToInternal::attributeRequired(this, name) and - instance_getattr(this, Types::getMro(this.getClass()), name, value, origin) + exists(ObjectInternal cls_attr, CfgOrigin attr_orig | + this.getClass().(ClassObjectInternal).lookup(name, cls_attr, attr_orig) + | + cls_attr.isDescriptor() = false and value = cls_attr and origin = attr_orig + or + cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this, value, origin) + ) or exists(EssaVariable self, PythonFunctionObjectInternal init, Context callee | BaseFlow::reaches_exit(self) and @@ -93,18 +100,21 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { ) } - override predicate attributesUnknown() { any() } + pragma [noinline] override predicate attributesUnknown() { any() } override boolean isDescriptor() { result = false } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { - this = instance and descriptor.isDescriptor() = true and - exists(AttrNode attr | - PointsToInternal::pointsTo(attr.getObject(name), _, instance, _) and - this.getClass().(ClassObjectInternal).lookup(name, descriptor, _) - ) + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { + exists(ClassObjectInternal cls | + receiver_type(_, name, this, cls) and + cls.lookup(name, descriptor, _) and + descriptor.isDescriptor() = true + ) and + this = instance } override int length() { @@ -113,18 +123,6 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { } -bindingset[instance, mro, name] -predicate instance_getattr(ObjectInternal instance, ClassList mro, string name, ObjectInternal value, CfgOrigin origin) { - exists(ObjectInternal descriptor, CfgOrigin desc_origin | - Types::declaredAttribute(mro.findDeclaringClass(name), name, descriptor, desc_origin) | - descriptor.isDescriptor() = false and - value = descriptor and origin = desc_origin - or - descriptor.isDescriptor() = true and - descriptor.descriptorGet(instance, value, origin) - ) -} - class SelfInstanceInternal extends TSelfInstance, ObjectInternal { @@ -159,19 +157,10 @@ class SelfInstanceInternal extends TSelfInstance, ObjectInternal { this = TSelfInstance(_, _, result) } - /** Gets the `Builtin` for this object, if any. - * All objects (except unknown and undefined values) should return - * exactly one result for either this method or `getOrigin()`. - */ override Builtin getBuiltin() { none() } - /** Gets a control flow node that represents the source origin of this - * objects. - * All objects (except unknown and undefined values) should return - * exactly one result for either this method or `getBuiltin()`. - */ override ControlFlowNode getOrigin() { exists(ParameterDefinition def | this = TSelfInstance(def, _, _) and @@ -201,18 +190,26 @@ class SelfInstanceInternal extends TSelfInstance, ObjectInternal { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + pragma [nomagic] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { PointsToInternal::attributeRequired(this, name) and - instance_getattr(this, Types::getMro(this.getClass()), name, value, origin) + exists(ObjectInternal cls_attr, CfgOrigin attr_orig | + this.getClass().(ClassObjectInternal).lookup(name, cls_attr, attr_orig) + | + cls_attr.isDescriptor() = false and value = cls_attr and origin = attr_orig + or + cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this, value, origin) + ) } - override predicate attributesUnknown() { any() } + pragma [noinline] override predicate attributesUnknown() { any() } override boolean isDescriptor() { result = false } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { exists(AttrNode attr, ClassObjectInternal cls | receiver_type(attr, name, this, cls) and cls_descriptor(cls, name, descriptor) @@ -295,18 +292,26 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { PointsToInternal::attributeRequired(this, name) and - instance_getattr(this, Types::getMro(this.getClass()), name, value, origin) + exists(ObjectInternal cls_attr, CfgOrigin attr_orig | + this.getClass().(ClassObjectInternal).lookup(name, cls_attr, attr_orig) + | + cls_attr.isDescriptor() = false and value = cls_attr and origin = attr_orig + or + cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this, value, origin) + ) } - override predicate attributesUnknown() { any() } + pragma [noinline] override predicate attributesUnknown() { any() } override boolean isDescriptor() { result = false } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { exists(AttrNode attr, ClassObjectInternal cls | receiver_type(attr, name, this, cls) and cls_descriptor(cls, name, descriptor) @@ -324,10 +329,6 @@ private int lengthFromClass(ClassObjectInternal cls) { Types::getMro(cls).declares("__len__") and result = -1 } -private predicate receiver_type(AttrNode attr, string name, ObjectInternal value, ClassObjectInternal cls) { - PointsToInternal::pointsTo(attr.getObject(name), _, value, _) and value.getClass() = cls -} - private predicate cls_descriptor(ClassObjectInternal cls, string name, ObjectInternal descriptor) { cls.lookup(name, descriptor, _) and descriptor.isDescriptor() = true @@ -361,7 +362,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal { override boolean isClass() { result = false } override ObjectInternal getClass() { - result = ObjectInternal::builtin("super") + result = ObjectInternal::super_() } override boolean isComparable() { result = false } @@ -382,28 +383,34 @@ class SuperInstance extends TSuperInstance, ObjectInternal { override predicate calleeAndOffset(Function scope, int paramOffset) { none() } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } override boolean isDescriptor() { result = false } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { PointsToInternal::attributeRequired(this, name) and - instance_getattr(this.getSelf(), this.getMro(), name, value, origin) + exists(ObjectInternal cls_attr, CfgOrigin attr_orig | + this.lookup(name, cls_attr, attr_orig) + | + cls_attr.isDescriptor() = false and value = cls_attr and origin = attr_orig + or + cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this.getSelf(), value, origin) + ) } - private ClassList getMro() { - result = Types::getMro(this.getSelf().getClass()).startingAt(this.getStartClass()).getTail() + private predicate lookup(string name, ObjectInternal value, CfgOrigin origin) { + Types::getMro(this.getSelf().getClass()).startingAt(this.getStartClass()).getTail().lookup(name, value, origin) } - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { descriptor.isDescriptor() = true and - exists(AttrNode attr | - PointsToInternal::pointsTo(attr.getObject(name), _, this, _) and - instance = this.getSelf() and - Types::declaredAttribute(this.getMro().findDeclaringClass(name), name, descriptor, _) - ) + this.lookup(name, descriptor, _) and + instance = this.getSelf() and + receiver_type(_, name, this, _) } override int length() { diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 897883d05d03..0de3b8d0bee7 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -37,10 +37,11 @@ abstract class ModuleObjectInternal extends ObjectInternal { override boolean isDescriptor() { result = false } + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } override int length() { none() } @@ -84,12 +85,12 @@ class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleOb none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { value = ObjectInternal::fromBuiltin(this.getBuiltin().getMember(name)) and origin = CfgOrigin::unknown() } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } override ControlFlowNode getOrigin() { none() @@ -154,7 +155,7 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { this.getInitModule().attribute(name, value, origin) or exists(Module init | @@ -176,7 +177,7 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { ) } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } override ControlFlowNode getOrigin() { result = this.getSourceModule().getEntryNode() @@ -234,19 +235,11 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - exists(EssaVariable var, ControlFlowNode exit, PointsToContext imp | - exit = this.getSourceModule().getANormalExit() and var.getAUse() = exit and - var.getSourceVariable().getName() = name and - PointsTo::variablePointsTo(var, imp, value, origin) and - imp.isImport() and - value != ObjectInternal::undefined() - ) - or + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { ModuleAttributes::pointsToAtExit(this.getSourceModule(), name, value, origin) } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } override ControlFlowNode getOrigin() { result = this.getSourceModule().getEntryNode() diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 9eb2cc4f8d26..37789be9bb99 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -90,7 +90,11 @@ class ObjectInternal extends TObject { abstract boolean isDescriptor(); - abstract predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin); + pragma[nomagic] + abstract predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin); + + pragma[nomagic] + abstract predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin); /** Holds if attribute lookup on this object may "bind" `instance` to `descriptor`. * Here "bind" means that `instance` is passed to the `descriptor.__get__()` method @@ -163,18 +167,20 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { value = ObjectInternal::fromBuiltin(this.getBuiltin().getMember(name)) and origin = CfgOrigin::unknown() } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } override boolean isDescriptor() { result = false } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } override int length() { none() } @@ -235,17 +241,19 @@ class UnknownInternal extends ObjectInternal, TUnknown { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } - override predicate attributesUnknown() { any() } + pragma [noinline] override predicate attributesUnknown() { any() } override boolean isDescriptor() { result = false } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } override int length() { result = -1 } @@ -307,17 +315,19 @@ class UndefinedInternal extends ObjectInternal, TUndefined { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } override boolean isDescriptor() { none() } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } override int length() { none() } @@ -390,9 +400,21 @@ module ObjectInternal { ObjectInternal property() { result = TBuiltinClassObject(Builtin::special("property")) } + + ObjectInternal super_() { + result = TBuiltinClassObject(Builtin::special("super")) + } + } /** Helper for boolean predicates returning both `true` and `false` */ boolean maybe() { result = true or result = false } + +/** Helper for attributes */ +pragma [nomagic] +predicate receiver_type(AttrNode attr, string name, ObjectInternal value, ClassObjectInternal cls) { + PointsToInternal::pointsTo(attr.getObject(name), _, value, _) and value.getClass() = cls +} + diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index a0434cb0f2eb..0a5482a46e6d 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -25,9 +25,11 @@ abstract class SequenceObjectInternal extends ObjectInternal { override boolean isDescriptor() { result = false } - override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } - override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } } @@ -81,9 +83,9 @@ abstract class TupleObjectInternal extends SequenceObjectInternal { override predicate calleeAndOffset(Function scope, int paramOffset) { none() } - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } - override predicate attributesUnknown() { none() } + pragma [noinline] override predicate attributesUnknown() { none() } } diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index cbe3a963b649..7d9e0938092b 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -99,7 +99,7 @@ newtype TObject = } or TUnknownInstance(BuiltinClassObjectInternal cls) { - cls != ObjectInternal::builtin("super") and + cls != ObjectInternal::super_() and cls != ObjectInternal::builtin("bool") and cls != ObjectInternal::noneType() } @@ -171,14 +171,21 @@ predicate super_instantiation(CallNode instantiation, ObjectInternal self, Class pragma [noinline] private predicate super_2args(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext context) { - exists(ControlFlowNode func, ControlFlowNode arg0, ControlFlowNode arg1 | - call2(instantiation, func, arg0, arg1) and - PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("super"), _) and + exists(ControlFlowNode arg0, ControlFlowNode arg1 | + super_call2(instantiation, arg0, arg1, context) and PointsToInternal::pointsTo(arg0, context, startclass, _) and PointsToInternal::pointsTo(arg1, context, self, _) ) } +pragma [noinline] +private predicate super_call2(CallNode call, ControlFlowNode arg0, ControlFlowNode arg1, PointsToContext context) { + exists(ControlFlowNode func | + call2(call, func, arg0, arg1) and + PointsToInternal::pointsTo(func, context, ObjectInternal::super_(), _) + ) +} + pragma [noinline] private predicate super_noargs(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext context) { PointsToInternal::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("super"), _) and @@ -205,7 +212,7 @@ predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableOb receiver(instantiation, context, obj, name) | exists(ObjectInternal cls | cls = obj.getClass() and - cls != ObjectInternal::builtin("super") and + cls != ObjectInternal::super_() and cls.attribute(name, function, _) and self = obj ) diff --git a/python/ql/src/semmle/python/pointsto/MRO.qll b/python/ql/src/semmle/python/pointsto/MRO.qll index af6bed2e02ac..2b3e7fcd3b09 100644 --- a/python/ql/src/semmle/python/pointsto/MRO.qll +++ b/python/ql/src/semmle/python/pointsto/MRO.qll @@ -155,6 +155,13 @@ class ClassList extends TClassList { ) } + predicate lookup(string name, ObjectInternal value, CfgOrigin origin) { + exists(ClassObjectInternal decl | + decl = this.findDeclaringClass(name) | + Types::declaredAttribute(decl, name, value, origin) + ) + } + predicate declares(string name) { this.getHead().getClassDeclaration().declaresAttribute(name) or diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index c2485a79afca..4abcc4fa0be7 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -175,6 +175,13 @@ cached module PointsToInternal { reachableBlock(f.getBasicBlock(), context) } + cached predicate pointsToString(ControlFlowNode f, PointsToContext context, string value) { + exists(StringObjectInternal str | + PointsToInternal::pointsTo(f, context, str, _) and + str.strValue() = value + ) + } + private predicate points_to_candidate(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { use_points_to(f, context, value, origin) or @@ -202,12 +209,7 @@ cached module PointsToInternal { cached predicate attributeRequired(ObjectInternal obj, string name) { pointsTo(any(AttrNode a).getObject(name), _, obj, _) or - exists(CallNode call, PointsToContext ctx, StringObjectInternal nameobj | - pointsTo(call.getFunction(), ctx, ObjectInternal::builtin("getattr"), _) and - pointsTo(call.getArg(0), ctx, obj, _) and - pointsTo(call.getArg(1), ctx, nameobj, _) and - nameobj.strValue() = name - ) + Expressions::getattr_call(_, _, _, obj, name) } /* Holds if BasicBlock `b` is reachable, given the context `context`. */ @@ -774,25 +776,49 @@ module InterModulePointsTo { module InterProceduralPointsTo { + cached predicate call(CallNode call, PointsToContext caller, ObjectInternal value) { + PointsToInternal::pointsTo(call.getFunction(), caller, value, _) + } + + cached predicate callWithContext(CallNode call, PointsToContext caller, ObjectInternal value, PointsToContext callee) { + callee.fromCall(call, caller) and + PointsToInternal::pointsTo(call.getFunction(), caller, value, _) + } + pragma [noinline] - predicate call_points_to(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - exists(ObjectInternal func, CfgOrigin resultOrigin | - call_points_to_callee(f, context, func) and - origin = resultOrigin.asCfgNodeOrHere(f) + private predicate call_points_to_simple(CallNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + exists(ObjectInternal func | + call(f, context, func) | exists(PointsToContext callee | callee.fromCall(f, context) and - func.callResult(callee, value, resultOrigin) + func.callResult(callee, value, origin) ) or - func.callResult(value, resultOrigin) and + context.untrackableCall(f) and + value = ObjectInternal::unknown() and origin = CfgOrigin::unknown() + or + func.callResult(value, origin) and context.appliesTo(f) ) + } + + pragma [noinline] + predicate call_points_to(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + exists(ObjectInternal returnValue, CfgOrigin resultOrigin | + call_points_to_simple(f, context, returnValue, resultOrigin) + | + /* Either not a decorator, or we understand the return value */ + (returnValue != ObjectInternal::unknown() or not f.isDecoratorCall()) and + value = returnValue and origin = resultOrigin.asCfgNodeOrHere(f) + or + /* A decorator and we don't understand it. Use the original, undecorated value */ + f.isDecoratorCall() and returnValue = ObjectInternal::unknown() and + PointsToInternal::pointsTo(f.getArg(0), context, value, origin) + ) or - call_to_type(f, context) and - exists(ObjectInternal arg | - PointsToInternal::pointsTo(f.getArg(0), context, arg, _) and - value = arg.getClass() | + value = call_to_type(f, context) and + ( value.isBuiltin() and origin = f or origin = value.getOrigin() @@ -802,14 +828,13 @@ module InterProceduralPointsTo { } pragma [noinline] - private predicate call_to_type(CallNode f, PointsToContext context) { + private ObjectInternal call_to_type(CallNode f, PointsToContext context) { count(f.getArg(_)) = 1 and - PointsToInternal::pointsTo(f.getFunction(), context, ObjectInternal::builtin("type"), _) - } - - pragma [noinline] - private predicate call_points_to_callee(CallNode f, PointsToContext context, ObjectInternal callee) { - PointsToInternal::pointsTo(f.getFunction(), context, callee, _) + call(f, context, ObjectInternal::builtin("type")) and + exists(ObjectInternal arg | + PointsToInternal::pointsTo(f.getArg(0), context, arg, _) and + result = arg.getClass() + ) } /** Points-to for parameter. `def foo(param): ...`. */ @@ -838,11 +863,11 @@ module InterProceduralPointsTo { context.isRuntime() and value = ObjectInternal::unknown() and origin = def.getDefiningNode() } + pragma [noinline] private predicate self_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { def.isSelf() and exists(CallNode call, BoundMethodObjectInternal method, Function func, PointsToContext caller | - PointsToInternal::pointsTo(call.getFunction(), caller, method, _) and - context.fromCall(call, caller) and + callWithContext(call, caller, method, context) and func = method.getScope() and def.getScope() = func and value = method.getSelf() and @@ -850,6 +875,16 @@ module InterProceduralPointsTo { ) } + predicate selfMethodCall(MethodCallsiteRefinement def, PointsToContext caller, Function func, PointsToContext callee) { + def.getInput().getSourceVariable().(Variable).isSelf() and + exists(PythonFunctionObjectInternal method, CallNode call | + method.getScope() = func and + call = method.getACall() and + call = def.getDefiningNode() and + callee.fromCall(call, caller) + ) + } + /** Helper for `parameter_points_to` */ pragma [noinline] private predicate named_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { @@ -861,6 +896,7 @@ module InterProceduralPointsTo { } /** Helper for parameter_points_to */ + pragma [noinline] private predicate default_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(PointsToContext imp | imp.isImport() | PointsToInternal::pointsTo(def.getDefault(), imp, value, origin)) and context_for_default_value(def, context) @@ -914,9 +950,8 @@ module InterProceduralPointsTo { } cached predicate callsite_calls_function(CallNode call, PointsToContext caller, Function scope, PointsToContext callee, int parameter_offset) { - callee.fromCall(call, caller) and exists(ObjectInternal func | - PointsToInternal::pointsTo(call.getFunction(), caller, func, _) and + callWithContext(call, caller, func, callee) and func.calleeAndOffset(scope, parameter_offset) ) } @@ -1049,11 +1084,16 @@ private predicate potential_builtin_points_to(NameNode f, ObjectInternal value, module Expressions { + pragma [noinline] + private predicate attributeObjectPointsto(AttrNode attr, PointsToContext context, string name, ControlFlowNode obj, ObjectInternal objvalue) { + attr.isLoad() and attr.getObject(name) = obj and + PointsToInternal::pointsTo(obj, context, objvalue, _) + } + + pragma [noinline] predicate attributePointsTo(AttrNode attr, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode obj, ObjectInternal objvalue) { - attr.isLoad() and exists(string name | - attr.getObject(name) = obj and - PointsToInternal::pointsTo(obj, context, objvalue, _) + attributeObjectPointsto(attr, context, name, obj, objvalue) | exists(CfgOrigin orig | objvalue.attribute(name, value, orig) and @@ -1061,10 +1101,12 @@ module Expressions { ) or objvalue.attributesUnknown() and - origin = attr and value = ObjectInternal::unknown() + origin = attr and + value = ObjectInternal::unknown() ) } + pragma [noinline] predicate subscriptPointsTo(SubscriptNode subscr, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode obj, ObjectInternal objvalue) { subscr.isLoad() and obj = subscr.getObject() and @@ -1075,6 +1117,7 @@ module Expressions { /** Track bitwise expressions so we can handle integer flags and enums. * Tracking too many binary expressions is likely to kill performance. */ + pragma [noinline] predicate binaryPointsTo(BinaryExprNode b, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode operand, ObjectInternal opvalue) { // TO DO... // Track some integer values through `|` and the types of some objects @@ -1083,6 +1126,7 @@ module Expressions { value = ObjectInternal::unknown() and origin = b } + pragma [noinline] predicate unaryPointsTo(UnaryExprNode u, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode operand, ObjectInternal opvalue) { exists(Unaryop op | op = u.getNode().getOp() and @@ -1098,6 +1142,7 @@ module Expressions { origin = u } + pragma [noinline] predicate builtinCallPointsTo(CallNode call, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode arg, ObjectInternal argvalue) { PointsToInternal::pointsTo(arg, context, argvalue, _) and arg = call.getArg(0) and @@ -1127,6 +1172,37 @@ module Expressions { ) } + pragma [noinline] + private predicate getattrPointsTo(CallNode call, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode arg, ObjectInternal argvalue) { + exists(string name | + getattr_call(call, arg, context, argvalue, name) + | + argvalue.attributesUnknown() and value = ObjectInternal::unknown() and origin = call + or + exists(CfgOrigin valOrigin | + argvalue.attribute(name, value, valOrigin) and origin = valOrigin.asCfgNodeOrHere(call) + ) + ) + } + + pragma [noinline] + predicate getattr_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, string name) { + exists(ControlFlowNode arg1 | + call_to_getattr(call, context, use, arg1) and + PointsToInternal::pointsTo(use, context, val, _) and + PointsToInternal::pointsToString(arg1, context, name) + ) + } + + pragma[noinline] + private predicate call_to_getattr(ControlFlowNode call, PointsToContext context, ControlFlowNode arg0, ControlFlowNode arg1) { + exists(ControlFlowNode func | + call2(call, func, arg0, arg1) and + PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("getattr"), _) + ) + } + + pragma [noinline] private boolean otherComparisonEvaluatesTo(CompareNode comp, PointsToContext context, ControlFlowNode operand, ObjectInternal opvalue) { exists(Cmpop op | comp.operands(operand, op, _) or @@ -1139,58 +1215,68 @@ module Expressions { pragma [noinline] private boolean equalityEvaluatesTo(CompareNode comp, PointsToContext context, ControlFlowNode operand, ObjectInternal opvalue) { - exists(ControlFlowNode r, boolean sense | - equality_test(comp, operand, sense, r) and - exists(ObjectInternal other | - PointsToInternal::pointsTo(operand, context, opvalue, _) and - PointsToInternal::pointsTo(r, context, other, _) | - opvalue.isComparable() = true and other.isComparable() = true and - ( - other = opvalue and result = sense - or - other != opvalue and result = sense.booleanNot() - ) - or - opvalue.isComparable() = false and result = maybe() + exists(ObjectInternal other, boolean sense | + equalityTest(comp, context, operand, opvalue, other, sense) + | + opvalue.isComparable() = true and other.isComparable() = true and + ( + other = opvalue and result = sense or - other.isComparable() = false and result = maybe() + other != opvalue and result = sense.booleanNot() ) + or + opvalue.isComparable() = false and result = maybe() + or + other.isComparable() = false and result = maybe() ) } pragma [noinline] - private boolean inequalityEvaluatesTo(ControlFlowNode expr, PointsToContext context, ControlFlowNode use, ObjectInternal val) { - exists(ControlFlowNode r, boolean sense | - exists(boolean strict, ObjectInternal other | - ( - inequality(expr, use, r, strict) and sense = true - or - inequality(expr, r, use, strict) and sense = false - ) and - PointsToInternal::pointsTo(use, context, val, _) and - PointsToInternal::pointsTo(r, context, other, _) - | - val.intValue() < other.intValue() and result = sense - or - val.intValue() > other.intValue() and result = sense.booleanNot() - or - val.intValue() = other.intValue() and result = strict.booleanXor(sense) - or - val.strValue() < other.strValue() and result = sense - or - val.strValue() > other.strValue() and result = sense.booleanNot() - or - val.strValue() = other.strValue() and result = strict.booleanXor(sense) - or - val.isComparable() = false and result = maybe() - or - other.isComparable() = false and result = maybe() - ) + private predicate equalityTest(CompareNode comp, PointsToContext context, ControlFlowNode operand, ObjectInternal opvalue, ObjectInternal other, boolean sense) { + exists(ControlFlowNode r | + equality_test(comp, operand, sense, r) and + PointsToInternal::pointsTo(operand, context, opvalue, _) and + PointsToInternal::pointsTo(r, context, other, _) + ) + } + + pragma [noinline] + private boolean inequalityEvaluatesTo(CompareNode comp, PointsToContext context, ControlFlowNode use, ObjectInternal val) { + exists(boolean strict, boolean sense, ObjectInternal other | + inequalityTest(comp, context, use, val, other, strict, sense) + | + val.intValue() < other.intValue() and result = sense + or + val.intValue() > other.intValue() and result = sense.booleanNot() + or + val.intValue() = other.intValue() and result = strict.booleanXor(sense) + or + val.strValue() < other.strValue() and result = sense + or + val.strValue() > other.strValue() and result = sense.booleanNot() + or + val.strValue() = other.strValue() and result = strict.booleanXor(sense) + or + val.isComparable() = false and result = maybe() + or + other.isComparable() = false and result = maybe() + ) + } + pragma [noinline] + private predicate inequalityTest(CompareNode comp, PointsToContext context, ControlFlowNode operand, ObjectInternal opvalue, ObjectInternal other, boolean strict, boolean sense) { + exists(ControlFlowNode r | + inequality(comp, operand, r, strict) and sense = true + or + inequality(comp, r, operand, strict) and sense = false + | + PointsToInternal::pointsTo(operand, context, opvalue, _) and + PointsToInternal::pointsTo(r, context, other, _) ) } /** Helper for comparisons. */ + pragma [noinline] private predicate inequality(CompareNode cmp, ControlFlowNode lesser, ControlFlowNode greater, boolean strict) { exists(Cmpop op | cmp.operands(lesser, op, greater) and op.getSymbol() = "<" and strict = true @@ -1216,9 +1302,12 @@ module Expressions { or lenCallPointsTo(expr, context, value, origin, subexpr, subvalue) or + getattrPointsTo(expr, context, value, origin, subexpr, subvalue) + or value = ObjectInternal::bool(evaluatesTo(expr, context, subexpr, subvalue)) and origin = expr } + pragma [noinline] boolean evaluatesTo(ControlFlowNode expr, PointsToContext context, ControlFlowNode subexpr, ObjectInternal subvalue) { result = equalityEvaluatesTo(expr, context, subexpr, subvalue) or @@ -1293,20 +1382,19 @@ module Expressions { } private predicate hasattr_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, string name) { - exists(ControlFlowNode func, ControlFlowNode arg1 | - call2(call, func, use, arg1) and - points_to_hasattr(func, context) and + exists(ControlFlowNode arg1 | + call_to_hasattr(call, context, use, arg1) and PointsToInternal::pointsTo(use, context, val, _) and - exists(StringObjectInternal str | - PointsToInternal::pointsTo(arg1, context, str, _) and - str.strValue() = name - ) + PointsToInternal::pointsToString(arg1, context, name) ) } pragma[noinline] - private predicate points_to_hasattr(ControlFlowNode func, PointsToContext context) { - PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("hasattr"), _) + private predicate call_to_hasattr(ControlFlowNode call, PointsToContext context, ControlFlowNode arg0, ControlFlowNode arg1) { + exists(ControlFlowNode func | + call2(call, func, arg0, arg1) and + PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("hasattr"), _) + ) } pragma [nomagic] @@ -1759,7 +1847,7 @@ module AttributePointsTo { private predicate selfParameterAttributePointsTo(ParameterDefinition def, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { exists(MethodCallsiteRefinement call, Function func, PointsToContext caller | - selfMethodCall(call, caller, func, context) and + InterProceduralPointsTo::selfMethodCall(call, caller, func, context) and def.isSelf() and def.getScope() = func and variableAttributePointsTo(call.getInput(), caller, name, value, origin) ) @@ -1769,7 +1857,7 @@ module AttributePointsTo { private predicate selfMethodCallsitePointsTo(MethodCallsiteRefinement def, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { /* The value of self remains the same, only the attributes may change */ exists(Function func, PointsToContext callee, EssaVariable exit_self | - selfMethodCall(def, context, func, callee) and + InterProceduralPointsTo::selfMethodCall(def, context, func, callee) and exit_self.getSourceVariable().(Variable).isSelf() and exit_self.getScope() = func and BaseFlow::reaches_exit(exit_self) and @@ -1777,16 +1865,6 @@ module AttributePointsTo { ) } - private predicate selfMethodCall(MethodCallsiteRefinement def, PointsToContext caller, Function func, PointsToContext callee) { - def.getInput().getSourceVariable().(Variable).isSelf() and - exists(PythonFunctionObjectInternal method, CallNode call | - method.getScope() = func and - call = method.getACall() and - call = def.getDefiningNode() and - callee.fromCall(call, caller) - ) - } - } module ModuleAttributes { @@ -1819,7 +1897,7 @@ module ModuleAttributes { scopeEntryPointsTo(var.getDefinition(), name, value, origin) } - pragma [noinline] + pragma [nomagic] private predicate importStarPointsTo(ImportStarRefinement def, string name, ObjectInternal value, CfgOrigin origin) { def.getVariable().getName() = "$" and exists(ImportStarNode imp, ModuleObjectInternal mod | diff --git a/python/ql/src/semmle/python/pointsto/PointsToContext.qll b/python/ql/src/semmle/python/pointsto/PointsToContext.qll index c57ff45504a2..d2dfdd9dccf0 100644 --- a/python/ql/src/semmle/python/pointsto/PointsToContext.qll +++ b/python/ql/src/semmle/python/pointsto/PointsToContext.qll @@ -14,6 +14,7 @@ private int given_cost() { ) } +pragma [noinline] private int max_context_cost() { not py_flags_versioned("context.cost", _, _) and result = 7 or @@ -103,6 +104,7 @@ private int total_call_cost(CallNode call) { result = call_cost(call) + splay_cost(call) } +pragma [noinline] private int total_cost(CallNode call, PointsToContext ctx) { ctx.appliesTo(call) and result = total_call_cost(call) + context_cost(ctx) @@ -238,6 +240,7 @@ class PointsToContext extends TPointsToContext { } /** Holds if a call would be too expensive to create a new context for */ + pragma [nomagic] predicate untrackableCall(CallNode call) { total_cost(call, this) > max_context_cost() } From 989d58761730032c7c4ae75d1337ce215219c7ad Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 17 Apr 2019 11:37:33 +0100 Subject: [PATCH 056/128] Python points-to: Fix support for backwards compatible extensions. --- python/ql/src/semmle/python/Flow.qll | 4 +- .../ql/src/semmle/python/objects/Classes.qll | 2 +- .../ql/src/semmle/python/objects/Modules.qll | 4 - .../src/semmle/python/objects/ObjectAPI.qll | 6 +- .../semmle/python/objects/ObjectInternal.qll | 2 +- .../src/semmle/python/pointsto/PointsTo.qll | 8 +- .../ql/src/semmle/python/types/Extensions.qll | 93 ++++++++++++++++--- python/ql/src/semmle/python/types/Object.qll | 4 +- .../PointsTo/comparisons/PointsTo.expected | 54 +++++------ .../PointsTo/comparisons/PointsTo.ql | 2 +- .../PointsTo/customise/test.expected | 1 + .../PointsTo/extensions/Extend.expected | 3 + 12 files changed, 123 insertions(+), 60 deletions(-) diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll index 36838818fe86..1b70a3b16639 100755 --- a/python/ql/src/semmle/python/Flow.qll +++ b/python/ql/src/semmle/python/Flow.qll @@ -217,8 +217,8 @@ class ControlFlowNode extends @py_flow_node { this.pointsTo(_, value, _) } - /** Gets the value that this ControlFlowNode points-to. */ - Value pointsTo() { + /** Gets a value that this ControlFlowNode may points-to. */ + Value inferredValue() { this.pointsTo(_, result, _) } diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 111e1ca64e1c..1dce54a71ca8 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -202,7 +202,7 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { override boolean isComparable() { result = false } override Builtin getBuiltin() { - none() + result = Builtin::unknownType() } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 0de3b8d0bee7..88489992766b 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -180,10 +180,6 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { pragma [noinline] override predicate attributesUnknown() { none() } override ControlFlowNode getOrigin() { - result = this.getSourceModule().getEntryNode() - } - - override @py_object getSource() { exists(Module package | package.isPackage() and package.getPath() = this.getFolder() and diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 8d4127b04c7e..836f9ec65140 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -9,9 +9,9 @@ class ObjectSource = Object; class Value extends TObject { Value() { - not this = ObjectInternal::unknown() and - not this = ObjectInternal::unknownClass() and - not this = ObjectInternal::undefined() + this != ObjectInternal::unknown() and + this != ObjectInternal::unknownClass() and + this != ObjectInternal::undefined() } string toString() { diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 37789be9bb99..62bd11ba36e7 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -214,7 +214,7 @@ class UnknownInternal extends ObjectInternal, TUnknown { override boolean isComparable() { result = false } override Builtin getBuiltin() { - none() + result = Builtin::unknown() } override predicate callResult(ObjectInternal obj, CfgOrigin origin) { diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 4abcc4fa0be7..4de29479f787 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -6,6 +6,7 @@ private import semmle.python.pointsto.Filters private import semmle.python.pointsto.PointsToContext private import semmle.python.pointsto.MRO private import semmle.python.types.Builtins +private import semmle.python.types.Extensions /* Use this version for speed */ //library class CfgOrigin extends @py_object { @@ -125,7 +126,7 @@ module PointsTo { /* Backwards compatibility */ deprecated predicate points_to(ControlFlowNode f, PointsToContext context, Object obj, ClassObject cls, ControlFlowNode origin) { - exists(Value value | + exists(ObjectInternal value | PointsToInternal::pointsTo(f, context, value, origin) and cls = value.getClass().getSource() | obj = value.getSource() or @@ -200,9 +201,8 @@ cached module PointsToInternal { InterProceduralPointsTo::call_points_to(f, context, value, origin) or AttributePointsTo::pointsTo(f, context, value, origin) - // To do... More stuff here :) - // or - // f.(CustomPointsToFact).pointsTo(context, value, origin) + or + f.(PointsToExtension).pointsTo(context, value, origin) } /** Holds if the attribute `name` is required for `obj` */ diff --git a/python/ql/src/semmle/python/types/Extensions.qll b/python/ql/src/semmle/python/types/Extensions.qll index 9046320e9ec4..594a1d201b54 100644 --- a/python/ql/src/semmle/python/types/Extensions.qll +++ b/python/ql/src/semmle/python/types/Extensions.qll @@ -12,12 +12,26 @@ import python private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext +private import semmle.python.objects.TObject +private import semmle.python.objects.ObjectInternal + +abstract class PointsToExtension extends @py_flow_node { + + string toString() { none() } + + abstract predicate pointsTo(Context context, ObjectInternal value, ControlFlowNode origin); + +} + + +/* Legacy API */ /* Custom Facts. This extension mechanism allows you to add custom * sources of data to the points-to analysis. */ -abstract class CustomPointsToFact extends @py_flow_node { +/** DEPRECATED -- Use PointsToExtension instead */ +deprecated abstract class CustomPointsToFact extends @py_flow_node { string toString() { none() } @@ -28,7 +42,7 @@ abstract class CustomPointsToFact extends @py_flow_node { /* For backwards compatibility */ class FinalCustomPointsToFact = CustomPointsToFact; -abstract class CustomPointsToOriginFact extends CustomPointsToFact { +deprecated abstract class CustomPointsToOriginFact extends CustomPointsToFact { abstract predicate pointsTo(Object value, ClassObject cls); @@ -39,7 +53,7 @@ abstract class CustomPointsToOriginFact extends CustomPointsToFact { } /* Custom points-to fact with inferred class */ -abstract class CustomPointsToObjectFact extends CustomPointsToFact { +deprecated abstract class CustomPointsToObjectFact extends CustomPointsToFact { abstract predicate pointsTo(Object value); @@ -50,8 +64,8 @@ abstract class CustomPointsToObjectFact extends CustomPointsToFact { } -/** INTERNAL -- Do not use */ -abstract class CustomPointsToAttribute extends Object { +/** DEPRECATED -- Unsupported; do not use */ +deprecated abstract class CustomPointsToAttribute extends Object { abstract predicate attributePointsTo(string name, Object value, ClassObject cls, ControlFlowNode origin); @@ -60,40 +74,89 @@ abstract class CustomPointsToAttribute extends Object { /* An example */ /** Any variable iterating over range or xrange must be an integer */ -class RangeIterationVariableFact extends CustomPointsToFact { +class RangeIterationVariableFact extends PointsToExtension { RangeIterationVariableFact() { exists(For f, ControlFlowNode iterable | iterable.getBasicBlock().dominates(this.(ControlFlowNode).getBasicBlock()) and f.getIter().getAFlowNode() = iterable and f.getTarget().getAFlowNode() = this and - PointsTo::points_to(iterable, _, theRangeType(), _, _) + exists(ObjectInternal range | + PointsTo::pointsTo(iterable, _, range, _) and + range.getClass() = ObjectInternal::builtin("range") + ) ) } - override predicate pointsTo(Context context, Object value, ClassObject cls, ControlFlowNode origin) { - value = this and + override predicate pointsTo(Context context, ObjectInternal value, ControlFlowNode origin) { + value = TUnknownInstance(ObjectInternal::builtin("int")) and origin = this and - cls = theIntType() and context.appliesTo(this) } + } /* Python 3.6+ regex module constants */ -class ReModulePointToExtension extends CustomPointsToAttribute { +class ReModulePointToExtension extends PointsToExtension { + + string name; ReModulePointToExtension() { - this.(ModuleObject).getName() = "re" + exists(ModuleObjectInternal re | re.getName() = "re" and + PointsTo::pointsTo(this.(AttrNode).getObject(name), _, re, _) + ) } - override predicate attributePointsTo(string name, Object value, ClassObject cls, ControlFlowNode origin) { - exists(ModuleObject sre_constants | + override predicate pointsTo(Context context, ObjectInternal value, ControlFlowNode origin) { + value = Module::named("sre_constants").attr("SRE_FLAG_" + name) and + exists(ModuleObjectInternal sre_constants | sre_constants.getName() = "sre_constants" and - sre_constants.attributeRefersTo("SRE_FLAG_" + name, value, cls, origin) + sre_constants.attribute(name, value, CfgOrigin::fromCfgNode(origin)) ) + and context.appliesTo(this) } + } +private class BackwardCompatiblePointToExtension extends PointsToExtension { + + BackwardCompatiblePointToExtension() { + this instanceof CustomPointsToFact + } + override predicate pointsTo(Context context, ObjectInternal value, ControlFlowNode origin) { + exists(Object obj, ClassObject cls | + this.(CustomPointsToFact).pointsTo(context, obj, cls, origin) + | + value.getBuiltin() = obj + or + obj instanceof ControlFlowNode and + exists(ClassObjectInternal c | + c.getSource() = cls and + value = TUnknownInstance(c) + ) + ) + or + exists(ObjectInternal owner, string name | + PointsTo::pointsTo(this.(AttrNode).getObject(name), context, owner, _) and + additionalAttribute(owner, name, value, origin) + ) + } + +} + +private predicate additionalAttribute(ObjectInternal owner, string name, ObjectInternal value, ControlFlowNode origin) { + exists(Object obj, ClassObject cls | + owner.getSource().(CustomPointsToAttribute).attributePointsTo(name, obj, cls, origin) + | + value.getBuiltin() = obj + or + obj instanceof ControlFlowNode and + exists(ClassObjectInternal c | + c.getSource() = cls and + value = TUnknownInstance(c) + ) + ) +} diff --git a/python/ql/src/semmle/python/types/Object.qll b/python/ql/src/semmle/python/types/Object.qll index e60830d89c83..26392b0ffde8 100644 --- a/python/ql/src/semmle/python/types/Object.qll +++ b/python/ql/src/semmle/python/types/Object.qll @@ -508,7 +508,7 @@ class SuperBoundMethod extends Object { string name; SuperBoundMethod() { - this.(AttrNode).getObject(name).pointsTo().getClass() = Value::named("super") + this.(AttrNode).getObject(name).inferredValue().getClass() = Value::named("super") } override string toString() { @@ -518,7 +518,7 @@ class SuperBoundMethod extends Object { Object getFunction(string fname) { fname = name and exists(SuperInstance sup, BoundMethodObjectInternal m | - sup = this.(AttrNode).getObject(name).pointsTo() and + sup = this.(AttrNode).getObject(name).inferredValue() and sup.attribute(name, m, _) and result = m.getFunction().getSource() ) diff --git a/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected b/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected index 8387ec955739..edad5dbbda8b 100644 --- a/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected +++ b/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected @@ -1,27 +1,27 @@ -| 2 | True | -| 3 | False | -| 6 | True | -| 7 | True | -| 8 | True | -| 9 | False | -| 19 | False | -| 19 | True | -| 20 | False | -| 20 | True | -| 21 | False | -| 21 | True | -| 22 | False | -| 22 | True | -| 25 | False | -| 26 | False | -| 27 | True | -| 28 | True | -| 29 | False | -| 30 | True | -| 33 | False | -| 33 | True | -| 34 | True | -| 35 | False | -| 36 | False | -| 37 | True | -| 38 | True | +| 2 | bool True | +| 3 | bool False | +| 6 | bool True | +| 7 | bool True | +| 8 | bool True | +| 9 | bool False | +| 19 | bool False | +| 19 | bool True | +| 20 | bool False | +| 20 | bool True | +| 21 | bool False | +| 21 | bool True | +| 22 | bool False | +| 22 | bool True | +| 25 | bool False | +| 26 | bool False | +| 27 | bool True | +| 28 | bool True | +| 29 | bool False | +| 30 | bool True | +| 33 | bool False | +| 33 | bool True | +| 34 | bool True | +| 35 | bool False | +| 36 | bool False | +| 37 | bool True | +| 38 | bool True | diff --git a/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.ql b/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.ql index c5bf9286904a..804ae3a61dfc 100644 --- a/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.ql +++ b/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.ql @@ -6,5 +6,5 @@ from int line, ControlFlowNode f, Value v where any(ExprStmt s).getValue() = f.getNode() and line = f.getLocation().getStartLine() and - f = v.getAReferent() + f.pointsTo(v) select line, v diff --git a/python/ql/test/library-tests/PointsTo/customise/test.expected b/python/ql/test/library-tests/PointsTo/customise/test.expected index 739123d92c78..93c492427086 100644 --- a/python/ql/test/library-tests/PointsTo/customise/test.expected +++ b/python/ql/test/library-tests/PointsTo/customise/test.expected @@ -1,3 +1,4 @@ +WARNING: Type CustomPointsToOriginFact has been deprecated and may be removed in future (test.ql:8,27-51) | 9 | ControlFlowNode for has_type_int | Function has_type_int | builtin-class function | | 9 | ControlFlowNode for has_type_int() | has_type_int() | builtin-class int | | 9 | ControlFlowNode for x | has_type_int() | builtin-class int | diff --git a/python/ql/test/library-tests/PointsTo/extensions/Extend.expected b/python/ql/test/library-tests/PointsTo/extensions/Extend.expected index 1cb63564722b..c728fb742339 100644 --- a/python/ql/test/library-tests/PointsTo/extensions/Extend.expected +++ b/python/ql/test/library-tests/PointsTo/extensions/Extend.expected @@ -1,3 +1,6 @@ +WARNING: Type CustomPointsToAttribute has been deprecated and may be removed in future (Extend.ql:26,35-58) +WARNING: Type CustomPointsToObjectFact has been deprecated and may be removed in future (Extend.ql:41,32-56) +WARNING: Type CustomPointsToOriginFact has been deprecated and may be removed in future (Extend.ql:8,28-52) | test.py:4:1:4:3 | ControlFlowNode for one | int 1 | | test.py:5:1:5:3 | ControlFlowNode for two | int 2 | | test.py:8:1:8:1 | ControlFlowNode for IntegerLiteral | int 1 | From 2d4f64f2e594bba8fe7e651af429b8e9cc7b228a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 17 Apr 2019 11:43:33 +0100 Subject: [PATCH 057/128] Python legacy objects: fix function/method call. --- python/ql/src/semmle/python/types/FunctionObject.qll | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/ql/src/semmle/python/types/FunctionObject.qll b/python/ql/src/semmle/python/types/FunctionObject.qll index 55a30bd673d9..4490507b9699 100644 --- a/python/ql/src/semmle/python/types/FunctionObject.qll +++ b/python/ql/src/semmle/python/types/FunctionObject.qll @@ -47,14 +47,15 @@ abstract class FunctionObject extends Object { /** Gets a call-site from where this function is called as a function */ CallNode getAFunctionCall() { - result.getFunction() instanceof NameNode and - result = this.getACall() + result.getFunction().inferredValue() = theCallable() } /** Gets a call-site from where this function is called as a method */ CallNode getAMethodCall() { - result.getFunction() instanceof AttrNode and - result = this.getACall() + exists(BoundMethodObjectInternal bm | + result.getFunction().inferredValue() = bm and + bm.getFunction() = theCallable() + ) } /** Gets a call-site from where this function is called */ From ddc4ada130132dcf6d4a7e15ef4dcb261043667e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 17 Apr 2019 12:32:40 +0100 Subject: [PATCH 058/128] Python points-to: Handle subclassing of ABCs. --- .../ql/src/semmle/python/objects/Classes.qll | 13 ++++++ .../src/semmle/python/objects/Instances.qll | 2 +- .../ql/src/semmle/python/objects/TObject.qll | 2 + .../src/semmle/python/pointsto/PointsTo.qll | 28 ++++++++++++- .../PointsTo/global/Global.expected | 42 +++++++++---------- .../library-tests/PointsTo/global/Global.ql | 8 ++-- 6 files changed, 68 insertions(+), 27 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 1dce54a71ca8..435a99871274 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -62,6 +62,19 @@ abstract class ClassObjectInternal extends ObjectInternal { override int length() { none() } + boolean isIterableSubclass() { + this = ObjectInternal::builtin("list") and result = true + or + this = ObjectInternal::builtin("set") and result = true + or + this = ObjectInternal::builtin("dict") and result = true + or + this != ObjectInternal::builtin("list") and + this != ObjectInternal::builtin("set") and + this != ObjectInternal::builtin("dict") and + result = false + } + } class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject { diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index e3a851bf1e25..a5b999ae83c8 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -31,7 +31,7 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { override boolean isClass() { result = false } - override boolean isComparable() { result = false } + override boolean isComparable() { result = true } override ObjectInternal getClass() { this = TSpecificInstance(_, result, _) diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 7d9e0938092b..c49896e2fabc 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -159,6 +159,8 @@ predicate literal_instantiation(ControlFlowNode n, ClassObjectInternal cls, Poin or n instanceof DictNode and cls = ObjectInternal::builtin("dict") or + n instanceof SetNode and cls = ObjectInternal::builtin("set") + or n.getNode() instanceof ImaginaryLiteral and cls = ObjectInternal::builtin("complex") ) } diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 4de29479f787..f01e8f1b1470 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1746,10 +1746,23 @@ cached module Types { or mro.getItem(n) = sup and result = true or - mro.getItem(n) != sup and result = mroContains(mro, sup, n+1) + mro.getItem(n) = abc_to_concrete(sup) and result = true + or + mro.getItem(n) != sup and sup != AbstractBaseClass::named("Iterable") and + mro.getItem(n) != abc_to_concrete(sup) and result = mroContains(mro, sup, n+1) + or + sup = AbstractBaseClass::named("Iterable") and result = mro.getItem(n).isIterableSubclass() ) } + private ClassObjectInternal abc_to_concrete(ClassObjectInternal c) { + c = AbstractBaseClass::named("Sequence") and result = ObjectInternal::builtin("list") + or + c = AbstractBaseClass::named("Set") and result = ObjectInternal::builtin("set") + or + c = AbstractBaseClass::named("Mapping") and result = ObjectInternal::builtin("dict") + } + cached boolean hasAttr(ObjectInternal cls, string name) { result = mroHasAttr(Types::getMro(cls), name, 0) } @@ -1776,6 +1789,19 @@ cached module Types { } +module AbstractBaseClass { + + ClassObjectInternal named(string name) { + exists(ModuleObjectInternal m | + m.getName() = "_abcoll" + or + m.getName() = "_collections_abc" + | + m.attribute(name, result, _) + ) + } +} + module AttributePointsTo { predicate pointsTo(AttrNode f, Context context, ObjectInternal value, ControlFlowNode origin) { diff --git a/python/ql/test/library-tests/PointsTo/global/Global.expected b/python/ql/test/library-tests/PointsTo/global/Global.expected index f119830c37e9..2ae828efd7ef 100644 --- a/python/ql/test/library-tests/PointsTo/global/Global.expected +++ b/python/ql/test/library-tests/PointsTo/global/Global.expected @@ -1,30 +1,30 @@ | import | test.py:17:1:17:3 | ControlFlowNode for bar | Function bar | test.py:2:1:2:19 | ControlFlowNode for FunctionExpr | -| import | test.py:18:1:18:7 | ControlFlowNode for bar() | 21 | test.py:18:5:18:6 | ControlFlowNode for IntegerLiteral | -| import | test.py:19:1:19:13 | ControlFlowNode for bar() | 22 | test.py:19:5:19:6 | ControlFlowNode for IntegerLiteral | -| import | test.py:20:1:20:13 | ControlFlowNode for bar() | True | test.py:20:9:20:12 | ControlFlowNode for True | -| import | test.py:21:1:21:11 | ControlFlowNode for bar() | 24 | test.py:21:5:21:6 | ControlFlowNode for IntegerLiteral | -| import | test.py:22:1:22:13 | ControlFlowNode for bar() | 7 | test.py:22:7:22:7 | ControlFlowNode for IntegerLiteral | +| import | test.py:18:1:18:7 | ControlFlowNode for bar() | int 21 | test.py:18:5:18:6 | ControlFlowNode for IntegerLiteral | +| import | test.py:19:1:19:13 | ControlFlowNode for bar() | int 22 | test.py:19:5:19:6 | ControlFlowNode for IntegerLiteral | +| import | test.py:20:1:20:13 | ControlFlowNode for bar() | bool True | test.py:20:9:20:12 | ControlFlowNode for True | +| import | test.py:21:1:21:11 | ControlFlowNode for bar() | int 24 | test.py:21:5:21:6 | ControlFlowNode for IntegerLiteral | +| import | test.py:22:1:22:13 | ControlFlowNode for bar() | int 7 | test.py:22:7:22:7 | ControlFlowNode for IntegerLiteral | | runtime | test.py:4:5:4:5 | ControlFlowNode for b | None | test.py:2:14:2:17 | ControlFlowNode for None | | runtime | test.py:11:5:11:7 | ControlFlowNode for bar | Function bar | test.py:2:1:2:19 | ControlFlowNode for FunctionExpr | -| runtime | test.py:12:5:12:11 | ControlFlowNode for bar() | 11 | test.py:12:9:12:10 | ControlFlowNode for IntegerLiteral | -| runtime | test.py:13:5:13:17 | ControlFlowNode for bar() | 12 | test.py:13:9:13:10 | ControlFlowNode for IntegerLiteral | -| runtime | test.py:14:5:14:17 | ControlFlowNode for bar() | True | test.py:14:13:14:16 | ControlFlowNode for True | -| runtime | test.py:15:5:15:15 | ControlFlowNode for bar() | 14 | test.py:15:9:15:10 | ControlFlowNode for IntegerLiteral | -| test.py:12 from runtime | test.py:3:5:3:5 | ControlFlowNode for a | 11 | test.py:12:9:12:10 | ControlFlowNode for IntegerLiteral | +| runtime | test.py:12:5:12:11 | ControlFlowNode for bar() | int 11 | test.py:12:9:12:10 | ControlFlowNode for IntegerLiteral | +| runtime | test.py:13:5:13:17 | ControlFlowNode for bar() | int 12 | test.py:13:9:13:10 | ControlFlowNode for IntegerLiteral | +| runtime | test.py:14:5:14:17 | ControlFlowNode for bar() | bool True | test.py:14:13:14:16 | ControlFlowNode for True | +| runtime | test.py:15:5:15:15 | ControlFlowNode for bar() | int 14 | test.py:15:9:15:10 | ControlFlowNode for IntegerLiteral | +| test.py:12 from runtime | test.py:3:5:3:5 | ControlFlowNode for a | int 11 | test.py:12:9:12:10 | ControlFlowNode for IntegerLiteral | | test.py:12 from runtime | test.py:4:5:4:5 | ControlFlowNode for b | None | test.py:2:14:2:17 | ControlFlowNode for None | -| test.py:13 from runtime | test.py:3:5:3:5 | ControlFlowNode for a | 12 | test.py:13:9:13:10 | ControlFlowNode for IntegerLiteral | +| test.py:13 from runtime | test.py:3:5:3:5 | ControlFlowNode for a | int 12 | test.py:13:9:13:10 | ControlFlowNode for IntegerLiteral | | test.py:13 from runtime | test.py:4:5:4:5 | ControlFlowNode for b | None | test.py:13:13:13:16 | ControlFlowNode for None | -| test.py:14 from runtime | test.py:3:5:3:5 | ControlFlowNode for a | 13 | test.py:14:9:14:10 | ControlFlowNode for IntegerLiteral | -| test.py:14 from runtime | test.py:4:5:4:5 | ControlFlowNode for b | True | test.py:14:13:14:16 | ControlFlowNode for True | -| test.py:15 from runtime | test.py:3:5:3:5 | ControlFlowNode for a | 14 | test.py:15:9:15:10 | ControlFlowNode for IntegerLiteral | +| test.py:14 from runtime | test.py:3:5:3:5 | ControlFlowNode for a | int 13 | test.py:14:9:14:10 | ControlFlowNode for IntegerLiteral | +| test.py:14 from runtime | test.py:4:5:4:5 | ControlFlowNode for b | bool True | test.py:14:13:14:16 | ControlFlowNode for True | +| test.py:15 from runtime | test.py:3:5:3:5 | ControlFlowNode for a | int 14 | test.py:15:9:15:10 | ControlFlowNode for IntegerLiteral | | test.py:15 from runtime | test.py:4:5:4:5 | ControlFlowNode for b | '' | test.py:15:13:15:14 | ControlFlowNode for Str | -| test.py:18 from import | test.py:3:5:3:5 | ControlFlowNode for a | 21 | test.py:18:5:18:6 | ControlFlowNode for IntegerLiteral | +| test.py:18 from import | test.py:3:5:3:5 | ControlFlowNode for a | int 21 | test.py:18:5:18:6 | ControlFlowNode for IntegerLiteral | | test.py:18 from import | test.py:4:5:4:5 | ControlFlowNode for b | None | test.py:2:14:2:17 | ControlFlowNode for None | -| test.py:19 from import | test.py:3:5:3:5 | ControlFlowNode for a | 22 | test.py:19:5:19:6 | ControlFlowNode for IntegerLiteral | +| test.py:19 from import | test.py:3:5:3:5 | ControlFlowNode for a | int 22 | test.py:19:5:19:6 | ControlFlowNode for IntegerLiteral | | test.py:19 from import | test.py:4:5:4:5 | ControlFlowNode for b | None | test.py:19:9:19:12 | ControlFlowNode for None | -| test.py:20 from import | test.py:3:5:3:5 | ControlFlowNode for a | 23 | test.py:20:5:20:6 | ControlFlowNode for IntegerLiteral | -| test.py:20 from import | test.py:4:5:4:5 | ControlFlowNode for b | True | test.py:20:9:20:12 | ControlFlowNode for True | -| test.py:21 from import | test.py:3:5:3:5 | ControlFlowNode for a | 24 | test.py:21:5:21:6 | ControlFlowNode for IntegerLiteral | +| test.py:20 from import | test.py:3:5:3:5 | ControlFlowNode for a | int 23 | test.py:20:5:20:6 | ControlFlowNode for IntegerLiteral | +| test.py:20 from import | test.py:4:5:4:5 | ControlFlowNode for b | bool True | test.py:20:9:20:12 | ControlFlowNode for True | +| test.py:21 from import | test.py:3:5:3:5 | ControlFlowNode for a | int 24 | test.py:21:5:21:6 | ControlFlowNode for IntegerLiteral | | test.py:21 from import | test.py:4:5:4:5 | ControlFlowNode for b | '' | test.py:21:9:21:10 | ControlFlowNode for Str | -| test.py:22 from import | test.py:3:5:3:5 | ControlFlowNode for a | 3 | test.py:22:12:22:12 | ControlFlowNode for IntegerLiteral | -| test.py:22 from import | test.py:4:5:4:5 | ControlFlowNode for b | 7 | test.py:22:7:22:7 | ControlFlowNode for IntegerLiteral | +| test.py:22 from import | test.py:3:5:3:5 | ControlFlowNode for a | int 3 | test.py:22:12:22:12 | ControlFlowNode for IntegerLiteral | +| test.py:22 from import | test.py:4:5:4:5 | ControlFlowNode for b | int 7 | test.py:22:7:22:7 | ControlFlowNode for IntegerLiteral | diff --git a/python/ql/test/library-tests/PointsTo/global/Global.ql b/python/ql/test/library-tests/PointsTo/global/Global.ql index 77971b12b0c8..8aa0453645fd 100644 --- a/python/ql/test/library-tests/PointsTo/global/Global.ql +++ b/python/ql/test/library-tests/PointsTo/global/Global.ql @@ -3,12 +3,12 @@ import python import python -import semmle.python.pointsto.PointsTo2 -import semmle.python.pointsto.PointsToContext2 +import semmle.python.pointsto.PointsTo +import semmle.python.pointsto.PointsToContext import semmle.python.objects.ObjectInternal -from ControlFlowNode f, PointsToContext2 ctx, ObjectInternal obj, ControlFlowNode orig +from ControlFlowNode f, PointsToContext ctx, Value obj, ControlFlowNode orig where exists(ExprStmt s | s.getValue().getAFlowNode() = f) and -PointsTo2::points_to(f, ctx, obj, orig) +PointsTo::pointsTo(f, ctx, obj, orig) select ctx, f, obj.toString(), orig From 4145b1967152a1a7c8a3a65a67f8b7e36aeadae1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 17 Apr 2019 13:01:36 +0100 Subject: [PATCH 059/128] Python points-to: update some test results. --- .../src/semmle/python/pointsto/PointsTo.qll | 11 ++++-- .../PointsTo/inheritance/Declared.expected | 22 ++++++------ .../PointsTo/inheritance/Declared.ql | 7 ++-- .../PointsTo/inheritance/Lookup.expected | 26 +++++++------- .../PointsTo/inheritance/Lookup.ql | 7 ++-- .../PointsTo/inheritance/MetaClass.expected | 3 ++ .../library-tests/PointsTo/inheritance/Mro.ql | 2 +- .../PointsTo/instances/LocalPointsTo.expected | 0 .../PointsTo/instances/LocalPointsTo.ql | 10 ------ .../library-tests/PointsTo/instances/test.py | 16 --------- .../PointsTo/local/LocalPointsTo.expected | 36 +++++++++---------- .../PointsTo/local/LocalPointsTo.ql | 4 +-- 12 files changed, 65 insertions(+), 79 deletions(-) delete mode 100644 python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.expected delete mode 100644 python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.ql delete mode 100644 python/ql/test/library-tests/PointsTo/instances/test.py diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index f01e8f1b1470..8652e89c0740 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1533,8 +1533,15 @@ cached module Types { result.getBuiltin() = cls.getBuiltin().getBaseClass() and n = 0 or exists(Class pycls | - pycls = cls.(PythonClassObjectInternal).getScope() | - PointsToInternal::pointsTo(pycls.getBase(n).getAFlowNode(), _, result, _) + pycls = cls.(PythonClassObjectInternal).getScope() + | + exists(ObjectInternal base | + PointsToInternal::pointsTo(pycls.getBase(n).getAFlowNode(), _, base, _) + | + result = base and base != ObjectInternal::unknown() + or + base = ObjectInternal::unknown() and result = ObjectInternal::unknownClass() + ) or not exists(pycls.getABase()) and n = 0 and isNewStyle(cls) and result = ObjectInternal::builtin("object") diff --git a/python/ql/test/library-tests/PointsTo/inheritance/Declared.expected b/python/ql/test/library-tests/PointsTo/inheritance/Declared.expected index daf2d029e7fa..b04ee388c8b1 100644 --- a/python/ql/test/library-tests/PointsTo/inheritance/Declared.expected +++ b/python/ql/test/library-tests/PointsTo/inheritance/Declared.expected @@ -1,11 +1,11 @@ -| class Base | meth | Function meth | 3 | -| class Derived1 | meth | Function meth | 8 | -| class Derived2 | meth | Function meth | 13 | -| class Derived4 | meth | Function meth | 21 | -| class Derived5 | meth | Function meth | 26 | -| class Derived6 | meth | Function meth | 31 | -| class Missing1 | a | Function a | 49 | -| class Missing2 | b | Function b | 53 | -| class Missing3 | c | Function c | 57 | -| class Wrong1 | meth | Function meth | 37 | -| class Wrong2 | meth | Function meth | 42 | +| class Base | meth | Function Base.meth | 3 | +| class Derived1 | meth | Function Derived1.meth | 8 | +| class Derived2 | meth | Function Derived2.meth | 13 | +| class Derived4 | meth | Function Derived4.meth | 21 | +| class Derived5 | meth | Function Derived5.meth | 26 | +| class Derived6 | meth | Function Derived6.meth | 31 | +| class Missing1 | a | Function Missing1.a | 49 | +| class Missing2 | b | Function Missing2.b | 53 | +| class Missing3 | c | Function Missing3.c | 57 | +| class Wrong1 | meth | Function Wrong1.meth | 37 | +| class Wrong2 | meth | Function Wrong2.meth | 42 | diff --git a/python/ql/test/library-tests/PointsTo/inheritance/Declared.ql b/python/ql/test/library-tests/PointsTo/inheritance/Declared.ql index 890fe308ea46..aefdb8894bdb 100644 --- a/python/ql/test/library-tests/PointsTo/inheritance/Declared.ql +++ b/python/ql/test/library-tests/PointsTo/inheritance/Declared.ql @@ -1,7 +1,8 @@ import python import semmle.python.pointsto.PointsTo +import semmle.python.objects.ObjectInternal -from ClassObject cls, string name, PyFunctionObject f -where PointsTo::Types::class_declared_attribute(cls, name, f, _, _) -select cls.toString(), name, f.toString(), f.getFunction().getLocation().getStartLine() +from ClassObjectInternal cls, string name, PythonFunctionObjectInternal f +where Types::declaredAttribute(cls, name, f, _) +select cls.toString(), name, f.toString(), f.getScope().getLocation().getStartLine() diff --git a/python/ql/test/library-tests/PointsTo/inheritance/Lookup.expected b/python/ql/test/library-tests/PointsTo/inheritance/Lookup.expected index d64169b65517..2b3d7e12bd49 100644 --- a/python/ql/test/library-tests/PointsTo/inheritance/Lookup.expected +++ b/python/ql/test/library-tests/PointsTo/inheritance/Lookup.expected @@ -1,13 +1,13 @@ -| class Base | meth | Function meth | 3 | -| class Derived1 | meth | Function meth | 8 | -| class Derived2 | meth | Function meth | 13 | -| class Derived3 | meth | Function meth | 8 | -| class Derived4 | meth | Function meth | 21 | -| class Derived5 | meth | Function meth | 26 | -| class Derived6 | meth | Function meth | 31 | -| class Missing1 | a | Function a | 49 | -| class Missing2 | b | Function b | 53 | -| class Missing2 | meth | Function meth | 3 | -| class Missing3 | c | Function c | 57 | -| class Wrong1 | meth | Function meth | 37 | -| class Wrong2 | meth | Function meth | 42 | +| class Base | meth | Function Base.meth | 3 | +| class Derived1 | meth | Function Derived1.meth | 8 | +| class Derived2 | meth | Function Derived2.meth | 13 | +| class Derived3 | meth | Function Derived1.meth | 8 | +| class Derived4 | meth | Function Derived4.meth | 21 | +| class Derived5 | meth | Function Derived5.meth | 26 | +| class Derived6 | meth | Function Derived6.meth | 31 | +| class Missing1 | a | Function Missing1.a | 49 | +| class Missing2 | b | Function Missing2.b | 53 | +| class Missing2 | meth | Function Base.meth | 3 | +| class Missing3 | c | Function Missing3.c | 57 | +| class Wrong1 | meth | Function Wrong1.meth | 37 | +| class Wrong2 | meth | Function Wrong2.meth | 42 | diff --git a/python/ql/test/library-tests/PointsTo/inheritance/Lookup.ql b/python/ql/test/library-tests/PointsTo/inheritance/Lookup.ql index 5f229b8d0d02..bf020de465b0 100644 --- a/python/ql/test/library-tests/PointsTo/inheritance/Lookup.ql +++ b/python/ql/test/library-tests/PointsTo/inheritance/Lookup.ql @@ -1,7 +1,8 @@ import python import semmle.python.pointsto.PointsTo +import semmle.python.objects.ObjectInternal -from ClassObject cls, string name, PyFunctionObject f -where PointsTo::Types::class_attribute_lookup(cls, name, f, _, _) -select cls.toString(), name, f.toString(), f.getFunction().getLocation().getStartLine() +from ClassObjectInternal cls, string name, PythonFunctionObjectInternal f +where cls.lookup(name, f, _) +select cls.toString(), name, f.toString(), f.getScope().getLocation().getStartLine() diff --git a/python/ql/test/library-tests/PointsTo/inheritance/MetaClass.expected b/python/ql/test/library-tests/PointsTo/inheritance/MetaClass.expected index 0348b74c1fb4..6bdcc99296ef 100644 --- a/python/ql/test/library-tests/PointsTo/inheritance/MetaClass.expected +++ b/python/ql/test/library-tests/PointsTo/inheritance/MetaClass.expected @@ -5,5 +5,8 @@ | class Derived4 | builtin-class type | | class Derived5 | builtin-class type | | class Derived6 | builtin-class type | +| class Missing1 | builtin-class type | +| class Missing2 | builtin-class type | +| class Missing3 | builtin-class type | | class Wrong1 | builtin-class type | | class Wrong2 | builtin-class type | diff --git a/python/ql/test/library-tests/PointsTo/inheritance/Mro.ql b/python/ql/test/library-tests/PointsTo/inheritance/Mro.ql index 1b3543a23519..1fe448b75293 100644 --- a/python/ql/test/library-tests/PointsTo/inheritance/Mro.ql +++ b/python/ql/test/library-tests/PointsTo/inheritance/Mro.ql @@ -2,7 +2,7 @@ import python private import semmle.python.objects.ObjectInternal -private import semmle.python.pointsto.PointsTo2 +private import semmle.python.pointsto.PointsTo /** Make unknown type visible */ class UnknownType extends UnknownClassInternal { diff --git a/python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.expected b/python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.expected deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.ql b/python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.ql deleted file mode 100644 index 8b14766b5c48..000000000000 --- a/python/ql/test/library-tests/PointsTo/instances/LocalPointsTo.ql +++ /dev/null @@ -1,10 +0,0 @@ - -import python -import semmle.python.pointsto.PointsTo2 -import semmle.python.objects.ObjectInternal - -from ControlFlowNode f, ObjectInternal obj, ControlFlowNode orig -where // exists(ExprStmt s | s.getValue().getAFlowNode() = f) and -PointsTo2::points_to(f, _, obj, orig) - -select f, obj.toString(), orig diff --git a/python/ql/test/library-tests/PointsTo/instances/test.py b/python/ql/test/library-tests/PointsTo/instances/test.py deleted file mode 100644 index 3b8117b6f6f8..000000000000 --- a/python/ql/test/library-tests/PointsTo/instances/test.py +++ /dev/null @@ -1,16 +0,0 @@ - - -class Foo(object): - pass - -class Bar(object): - pass - -Foo -Bar - -f = Foo() -b = Bar() - -f -b diff --git a/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.expected b/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.expected index d4acab646d3a..3de409632f15 100644 --- a/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.expected +++ b/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.expected @@ -1,22 +1,22 @@ | test.py:7:1:7:1 | ControlFlowNode for n | None | test.py:3:5:3:8 | ControlFlowNode for None | -| test.py:8:1:8:1 | ControlFlowNode for t | True | test.py:4:5:4:8 | ControlFlowNode for True | -| test.py:9:1:9:1 | ControlFlowNode for f | False | test.py:5:5:5:9 | ControlFlowNode for False | -| test.py:14:5:14:5 | ControlFlowNode for f | False | test.py:5:5:5:9 | ControlFlowNode for False | +| test.py:8:1:8:1 | ControlFlowNode for t | bool True | test.py:4:5:4:8 | ControlFlowNode for True | +| test.py:9:1:9:1 | ControlFlowNode for f | bool False | test.py:5:5:5:9 | ControlFlowNode for False | +| test.py:14:5:14:5 | ControlFlowNode for f | bool False | test.py:5:5:5:9 | ControlFlowNode for False | | test.py:22:5:22:5 | ControlFlowNode for n | None | test.py:18:9:18:12 | ControlFlowNode for None | -| test.py:23:5:23:5 | ControlFlowNode for t | True | test.py:19:9:19:12 | ControlFlowNode for True | -| test.py:24:5:24:5 | ControlFlowNode for f | False | test.py:20:9:20:13 | ControlFlowNode for False | -| test.py:29:9:29:9 | ControlFlowNode for f | False | test.py:20:9:20:13 | ControlFlowNode for False | -| test.py:31:5:31:9 | ControlFlowNode for UnaryExpr | True | test.py:31:5:31:9 | ControlFlowNode for UnaryExpr | -| test.py:32:5:32:9 | ControlFlowNode for UnaryExpr | False | test.py:32:5:32:9 | ControlFlowNode for UnaryExpr | -| test.py:33:5:33:9 | ControlFlowNode for UnaryExpr | True | test.py:33:5:33:9 | ControlFlowNode for UnaryExpr | -| test.py:38:9:38:9 | ControlFlowNode for f | False | test.py:20:9:20:13 | ControlFlowNode for False | -| test.py:41:9:41:9 | ControlFlowNode for t | True | test.py:19:9:19:12 | ControlFlowNode for True | -| test.py:49:5:49:6 | ControlFlowNode for IntegerLiteral | 17 | test.py:49:5:49:6 | ControlFlowNode for IntegerLiteral | -| test.py:50:5:50:6 | ControlFlowNode for UnaryExpr | -3 | test.py:50:5:50:6 | ControlFlowNode for UnaryExpr | -| test.py:52:5:52:9 | ControlFlowNode for Compare | True | test.py:52:5:52:9 | ControlFlowNode for Compare | -| test.py:57:9:57:9 | ControlFlowNode for w | 2 | test.py:45:9:45:9 | ControlFlowNode for IntegerLiteral | -| test.py:58:9:58:9 | ControlFlowNode for z | 5 | test.py:48:9:48:9 | ControlFlowNode for IntegerLiteral | -| test.py:64:9:64:9 | ControlFlowNode for v | 20 | test.py:62:13:62:14 | ControlFlowNode for IntegerLiteral | -| test.py:66:9:66:9 | ControlFlowNode for v | 10 | test.py:60:13:60:14 | ControlFlowNode for IntegerLiteral | +| test.py:23:5:23:5 | ControlFlowNode for t | bool True | test.py:19:9:19:12 | ControlFlowNode for True | +| test.py:24:5:24:5 | ControlFlowNode for f | bool False | test.py:20:9:20:13 | ControlFlowNode for False | +| test.py:29:9:29:9 | ControlFlowNode for f | bool False | test.py:20:9:20:13 | ControlFlowNode for False | +| test.py:31:5:31:9 | ControlFlowNode for UnaryExpr | bool True | test.py:31:5:31:9 | ControlFlowNode for UnaryExpr | +| test.py:32:5:32:9 | ControlFlowNode for UnaryExpr | bool False | test.py:32:5:32:9 | ControlFlowNode for UnaryExpr | +| test.py:33:5:33:9 | ControlFlowNode for UnaryExpr | bool True | test.py:33:5:33:9 | ControlFlowNode for UnaryExpr | +| test.py:38:9:38:9 | ControlFlowNode for f | bool False | test.py:20:9:20:13 | ControlFlowNode for False | +| test.py:41:9:41:9 | ControlFlowNode for t | bool True | test.py:19:9:19:12 | ControlFlowNode for True | +| test.py:49:5:49:6 | ControlFlowNode for IntegerLiteral | int 17 | test.py:49:5:49:6 | ControlFlowNode for IntegerLiteral | +| test.py:50:5:50:6 | ControlFlowNode for UnaryExpr | int -3 | test.py:50:5:50:6 | ControlFlowNode for UnaryExpr | +| test.py:52:5:52:9 | ControlFlowNode for Compare | bool True | test.py:52:5:52:9 | ControlFlowNode for Compare | +| test.py:57:9:57:9 | ControlFlowNode for w | int 2 | test.py:45:9:45:9 | ControlFlowNode for IntegerLiteral | +| test.py:58:9:58:9 | ControlFlowNode for z | int 5 | test.py:48:9:48:9 | ControlFlowNode for IntegerLiteral | +| test.py:64:9:64:9 | ControlFlowNode for v | int 20 | test.py:62:13:62:14 | ControlFlowNode for IntegerLiteral | +| test.py:66:9:66:9 | ControlFlowNode for v | int 10 | test.py:60:13:60:14 | ControlFlowNode for IntegerLiteral | | test.py:70:5:70:14 | ControlFlowNode for Str | 'a string' | test.py:70:5:70:14 | ControlFlowNode for Str | | test.py:72:9:72:12 | ControlFlowNode for Str | ':)' | test.py:72:9:72:12 | ControlFlowNode for Str | diff --git a/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.ql b/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.ql index 767736718626..d3cde3b0771d 100644 --- a/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.ql +++ b/python/ql/test/library-tests/PointsTo/local/LocalPointsTo.ql @@ -1,10 +1,10 @@ import python -import semmle.python.pointsto.PointsTo2 +import semmle.python.pointsto.PointsTo import semmle.python.objects.ObjectInternal from ControlFlowNode f, ObjectInternal obj, ControlFlowNode orig where exists(ExprStmt s | s.getValue().getAFlowNode() = f) and -PointsTo2::points_to(f, _, obj, orig) +PointsTo::pointsTo(f, _, obj, orig) select f, obj.toString(), orig From 2aa967ae8ebfed71f7089d253e84f2c1ab298d5c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 17 Apr 2019 13:04:35 +0100 Subject: [PATCH 060/128] Remove redundant tests. --- .../src/semmle/python/pointsto/PointsTo.qll | 27 +++++++++++++------ .../semmle/python/security/TaintTracking.qll | 5 ++-- .../PointsTo/methods/PointsTo.expected | 9 ------- .../PointsTo/methods/PointsTo.ql | 10 ------- .../library-tests/PointsTo/methods/test.py | 27 ------------------- .../PointsTo/new/Values.expected | 1 + .../test/library-tests/PointsTo/new/Values.ql | 9 +++++++ .../taint/general/ModuleAttribute.ql | 2 +- 8 files changed, 32 insertions(+), 58 deletions(-) delete mode 100644 python/ql/test/library-tests/PointsTo/methods/PointsTo.expected delete mode 100644 python/ql/test/library-tests/PointsTo/methods/PointsTo.ql delete mode 100644 python/ql/test/library-tests/PointsTo/methods/test.py create mode 100644 python/ql/test/library-tests/PointsTo/new/Values.expected create mode 100644 python/ql/test/library-tests/PointsTo/new/Values.ql diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 8652e89c0740..0f0427cab326 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1705,16 +1705,10 @@ cached module Types { or exists(cls.(PythonClassObjectInternal).getScope().getADecorator()) and not six_add_metaclass(_, cls, _) and reason = "Decorator not understood" or - exists(int i | - exists(cls.(PythonClassObjectInternal).getScope().getBase(i)) and reason = "Missing base " + i - | - not exists(getBase(cls, i)) - ) + reason = "Missing base " + missingBase(cls) or exists(cls.(PythonClassObjectInternal).getScope().getMetaClass()) and not exists(cls.getClass()) and reason = "Failed to infer metaclass" or - exists(int i | failedInference(getBase(cls, i), _) and reason = "Failed inference for base class at position " + i) - or exists(int i, ObjectInternal base1, ObjectInternal base2 | base1 = getBase(cls, i) and base2 = getBase(cls, i) and @@ -1722,7 +1716,24 @@ cached module Types { reason = "Multiple bases at position " + i ) or - exists(int i, int j | getBase(cls, i) = getBase(cls, j) and i != j and reason = "Duplicate bases classes") + duplicateBase(cls) and reason = "Duplicate bases classes" + or + not exists(getMro(cls)) and reason = "Failed to compute MRO" and not exists(missingBase(cls)) and not duplicateBase(cls) + or + exists(int i | failedInference(getBase(cls, i), _) and reason = "Failed inference for base class at position " + i) + } + + private int missingBase(ClassObjectInternal cls) { + exists(cls.(PythonClassObjectInternal).getScope().getBase(result)) + and + not exists(getBase(cls, result)) or getBase(cls, result) = ObjectInternal::unknownClass() + } + + private predicate duplicateBase(ClassObjectInternal cls) { + exists(int i, int j, ClassObjectInternal dup | + dup = getBase(cls, i) and dup != ObjectInternal::unknownClass() and + dup = getBase(cls, j) and i != j + ) } cached boolean improperSubclass(ObjectInternal sub, ObjectInternal sup) { diff --git a/python/ql/src/semmle/python/security/TaintTracking.qll b/python/ql/src/semmle/python/security/TaintTracking.qll index b97b0d0861a3..ea225f39aac9 100755 --- a/python/ql/src/semmle/python/security/TaintTracking.qll +++ b/python/ql/src/semmle/python/security/TaintTracking.qll @@ -1204,11 +1204,10 @@ library module TaintFlowImplementation { exists(ParameterDefinition def | def.getDefiningNode() = param and exists(CallableValue func, CallNode call | - call.getFunction().pointsTo() = func and callee = caller.getCallee(call) | - exists(int n | param = func.getParameter(n) and argument = call.getArg(n)) + exists(int n | param = func.getParameter(n) and argument = func.getArgumentForCall(call, n)) or - exists(string name | param = func.getParameterByName(name) and argument = call.getArgByName(name)) + exists(string name | param = func.getParameterByName(name) and argument = func.getNamedArgumentForCall(call, name)) or class_initializer_argument(_, _, call, func, argument, param) ) diff --git a/python/ql/test/library-tests/PointsTo/methods/PointsTo.expected b/python/ql/test/library-tests/PointsTo/methods/PointsTo.expected deleted file mode 100644 index 0a6012419a07..000000000000 --- a/python/ql/test/library-tests/PointsTo/methods/PointsTo.expected +++ /dev/null @@ -1,9 +0,0 @@ -| test.py:10 ... fail -| test.py:11:5:11:7 | ControlFlowNode for Attribute | bound method 'm' of instance of C | test.py:11:5:11:7 | ControlFlowNode for Attribute | -| test.py:12:5:12:10 | ControlFlowNode for Attribute() | 3 | test.py:12:9:12:9 | ControlFlowNode for IntegerLiteral | -| test.py:16:5:16:5 | ControlFlowNode for t | bound method 'm' of instance of C | test.py:15:9:15:11 | ControlFlowNode for Attribute | -| test.py:17:5:17:8 | ControlFlowNode for t() | 4 | test.py:17:7:17:7 | ControlFlowNode for IntegerLiteral | -| test.py:24:1:24:1 | ControlFlowNode for D | class D | test.py:19:1:19:16 | ControlFlowNode for ClassExpr | -| test.py:25 ... fail -| test.py:26 ... fail -| test.py:27 ... fail diff --git a/python/ql/test/library-tests/PointsTo/methods/PointsTo.ql b/python/ql/test/library-tests/PointsTo/methods/PointsTo.ql deleted file mode 100644 index 767736718626..000000000000 --- a/python/ql/test/library-tests/PointsTo/methods/PointsTo.ql +++ /dev/null @@ -1,10 +0,0 @@ - -import python -import semmle.python.pointsto.PointsTo2 -import semmle.python.objects.ObjectInternal - -from ControlFlowNode f, ObjectInternal obj, ControlFlowNode orig -where exists(ExprStmt s | s.getValue().getAFlowNode() = f) and -PointsTo2::points_to(f, _, obj, orig) - -select f, obj.toString(), orig diff --git a/python/ql/test/library-tests/PointsTo/methods/test.py b/python/ql/test/library-tests/PointsTo/methods/test.py deleted file mode 100644 index 858545eb7856..000000000000 --- a/python/ql/test/library-tests/PointsTo/methods/test.py +++ /dev/null @@ -1,27 +0,0 @@ - -class C(object): - - def m(self, arg1): - return arg1 - -c = C() - -def func_and_method(): - C.m - c.m - c.m(3) - -def flow_bound_method(): - t = c.m - t - t(4) - -class D(object): - - @staticmethod - def foo(arg): - return arg -D -D.foo -D.foo(1) -D().foo(2) diff --git a/python/ql/test/library-tests/PointsTo/new/Values.expected b/python/ql/test/library-tests/PointsTo/new/Values.expected new file mode 100644 index 000000000000..9dae856c349e --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/new/Values.expected @@ -0,0 +1 @@ +fail diff --git a/python/ql/test/library-tests/PointsTo/new/Values.ql b/python/ql/test/library-tests/PointsTo/new/Values.ql new file mode 100644 index 000000000000..f1f3dd43aa49 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/new/Values.ql @@ -0,0 +1,9 @@ + +import python +import Util + + +from ControlFlowNode f, Context ctx, Value v, ControlFlowNode origin +where + f.pointsTo(ctx, v, origin) +select locate(f.getLocation(), "c"), f.toString(), ctx, v, v.getClass() diff --git a/python/ql/test/library-tests/taint/general/ModuleAttribute.ql b/python/ql/test/library-tests/taint/general/ModuleAttribute.ql index 6daca6cda1bf..3dbe9eed66d7 100644 --- a/python/ql/test/library-tests/taint/general/ModuleAttribute.ql +++ b/python/ql/test/library-tests/taint/general/ModuleAttribute.ql @@ -3,7 +3,7 @@ import semmle.python.security.TaintTest import TaintLib -from ModuleObject m, string name, TaintedNode origin +from ModuleValue m, string name, TaintedNode origin where TaintFlowTest::module_attribute_tainted(m, name, origin) From 2ea204f7f97da219c7ff2770ac7ccdb3fab72dcd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 17 Apr 2019 15:04:54 +0100 Subject: [PATCH 061/128] Python points-to: Fix up failedInference predicate. --- .../PointsTo/new/Values.expected | 874 +++++++++++++++++- .../test/library-tests/PointsTo/new/Values.ql | 2 +- .../types/classes/FailedInference.expected | 60 +- .../types/classes/FailedInference.ql | 4 +- 4 files changed, 911 insertions(+), 29 deletions(-) diff --git a/python/ql/test/library-tests/PointsTo/new/Values.expected b/python/ql/test/library-tests/PointsTo/new/Values.expected index 9dae856c349e..a388d1c2c897 100644 --- a/python/ql/test/library-tests/PointsTo/new/Values.expected +++ b/python/ql/test/library-tests/PointsTo/new/Values.expected @@ -1 +1,873 @@ -fail +| a_simple.py:2 | ControlFlowNode for FloatLiteral | import | float 1.0 | builtin-class float | +| a_simple.py:3 | ControlFlowNode for dict | import | builtin-class dict | builtin-class type | +| a_simple.py:4 | ControlFlowNode for tuple | import | builtin-class tuple | builtin-class type | +| a_simple.py:5 | ControlFlowNode for IntegerLiteral | import | int 0 | builtin-class int | +| a_simple.py:6 | ControlFlowNode for Tuple | import | () | builtin-class tuple | +| a_simple.py:8 | ControlFlowNode for FunctionExpr | import | Function func | builtin-class function | +| a_simple.py:11 | ControlFlowNode for ClassExpr | import | class C | builtin-class type | +| a_simple.py:11 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| a_simple.py:14 | ControlFlowNode for FunctionExpr | import | Function vararg_kwarg | builtin-class function | +| a_simple.py:15 | ControlFlowNode for t | runtime | instance of tuple | builtin-class tuple | +| a_simple.py:16 | ControlFlowNode for d | runtime | instance of dict | builtin-class dict | +| a_simple.py:18 | ControlFlowNode for FunctionExpr | import | Function multi_loop | builtin-class function | +| a_simple.py:19 | ControlFlowNode for None | runtime | None | builtin-class NoneType | +| a_simple.py:23 | ControlFlowNode for FunctionExpr | import | Function with_definition | builtin-class function | +| a_simple.py:27 | ControlFlowNode for FunctionExpr | import | Function multi_loop_in_try | builtin-class function | +| a_simple.py:31 | ControlFlowNode for KeyError | runtime | builtin-class KeyError | builtin-class type | +| a_simple.py:34 | ControlFlowNode for FunctionExpr | import | Function f | builtin-class function | +| a_simple.py:35 | ControlFlowNode for IntegerLiteral | runtime | int 0 | builtin-class int | +| a_simple.py:35 | ControlFlowNode for UnaryExpr | runtime | bool False | builtin-class bool | +| a_simple.py:35 | ControlFlowNode for UnaryExpr | runtime | bool True | builtin-class bool | +| a_simple.py:35 | ControlFlowNode for args | runtime | instance of tuple | builtin-class tuple | +| a_simple.py:36 | ControlFlowNode for Str | runtime | 'x' | builtin-class str | +| a_simple.py:36 | ControlFlowNode for UnaryExpr | runtime | bool False | builtin-class bool | +| a_simple.py:36 | ControlFlowNode for UnaryExpr | runtime | bool True | builtin-class bool | +| a_simple.py:36 | ControlFlowNode for kwargs | runtime | instance of dict | builtin-class dict | +| b_condition.py:4 | ControlFlowNode for FunctionExpr | import | Function f | builtin-class function | +| b_condition.py:5 | ControlFlowNode for IfExp | runtime | None | builtin-class NoneType | +| b_condition.py:5 | ControlFlowNode for None | runtime | None | builtin-class NoneType | +| b_condition.py:7 | ControlFlowNode for Compare | runtime | bool False | builtin-class bool | +| b_condition.py:7 | ControlFlowNode for Compare | runtime | bool True | builtin-class bool | +| b_condition.py:7 | ControlFlowNode for None | runtime | None | builtin-class NoneType | +| b_condition.py:7 | ControlFlowNode for x | runtime | None | builtin-class NoneType | +| b_condition.py:8 | ControlFlowNode for IntegerLiteral | runtime | int 7 | builtin-class int | +| b_condition.py:9 | ControlFlowNode for x | runtime | int 7 | builtin-class int | +| b_condition.py:11 | ControlFlowNode for IfExp | runtime | None | builtin-class NoneType | +| b_condition.py:11 | ControlFlowNode for None | runtime | None | builtin-class NoneType | +| b_condition.py:13 | ControlFlowNode for Compare | runtime | bool False | builtin-class bool | +| b_condition.py:13 | ControlFlowNode for Compare | runtime | bool True | builtin-class bool | +| b_condition.py:13 | ControlFlowNode for None | runtime | None | builtin-class NoneType | +| b_condition.py:13 | ControlFlowNode for x | runtime | None | builtin-class NoneType | +| b_condition.py:14 | ControlFlowNode for IntegerLiteral | runtime | int 7 | builtin-class int | +| b_condition.py:15 | ControlFlowNode for x | runtime | None | builtin-class NoneType | +| b_condition.py:15 | ControlFlowNode for x | runtime | int 7 | builtin-class int | +| b_condition.py:17 | ControlFlowNode for IfExp | runtime | None | builtin-class NoneType | +| b_condition.py:17 | ControlFlowNode for None | runtime | None | builtin-class NoneType | +| b_condition.py:19 | ControlFlowNode for UnaryExpr | runtime | bool False | builtin-class bool | +| b_condition.py:19 | ControlFlowNode for UnaryExpr | runtime | bool True | builtin-class bool | +| b_condition.py:19 | ControlFlowNode for x | runtime | None | builtin-class NoneType | +| b_condition.py:20 | ControlFlowNode for None | runtime | None | builtin-class NoneType | +| b_condition.py:21 | ControlFlowNode for x | runtime | None | builtin-class NoneType | +| b_condition.py:23 | ControlFlowNode for IfExp | runtime | None | builtin-class NoneType | +| b_condition.py:23 | ControlFlowNode for None | runtime | None | builtin-class NoneType | +| b_condition.py:25 | ControlFlowNode for IfExp | runtime | int 1 | builtin-class int | +| b_condition.py:25 | ControlFlowNode for IntegerLiteral | runtime | int 1 | builtin-class int | +| b_condition.py:25 | ControlFlowNode for x | runtime | None | builtin-class NoneType | +| b_condition.py:26 | ControlFlowNode for x | runtime | int 1 | builtin-class int | +| b_condition.py:28 | ControlFlowNode for IntegerLiteral | runtime | int 1 | builtin-class int | +| b_condition.py:29 | ControlFlowNode for x | runtime | int 1 | builtin-class int | +| b_condition.py:31 | ControlFlowNode for IfExp | runtime | int 1 | builtin-class int | +| b_condition.py:31 | ControlFlowNode for IntegerLiteral | runtime | int 1 | builtin-class int | +| b_condition.py:32 | ControlFlowNode for UnaryExpr | runtime | bool False | builtin-class bool | +| b_condition.py:32 | ControlFlowNode for UnaryExpr | runtime | bool True | builtin-class bool | +| b_condition.py:32 | ControlFlowNode for x | runtime | int 1 | builtin-class int | +| b_condition.py:33 | ControlFlowNode for IntegerLiteral | runtime | int 7 | builtin-class int | +| b_condition.py:34 | ControlFlowNode for x | runtime | int 1 | builtin-class int | +| b_condition.py:34 | ControlFlowNode for x | runtime | int 7 | builtin-class int | +| b_condition.py:36 | ControlFlowNode for int | runtime | builtin-class int | builtin-class type | +| b_condition.py:36 | ControlFlowNode for isinstance | runtime | Builtin-function isinstance | builtin-class builtin_function_or_method | +| b_condition.py:36 | ControlFlowNode for isinstance() | runtime | bool False | builtin-class bool | +| b_condition.py:36 | ControlFlowNode for isinstance() | runtime | bool True | builtin-class bool | +| b_condition.py:36 | ControlFlowNode for x | runtime | int 1 | builtin-class int | +| b_condition.py:36 | ControlFlowNode for x | runtime | int 7 | builtin-class int | +| b_condition.py:37 | ControlFlowNode for x | runtime | int 1 | builtin-class int | +| b_condition.py:37 | ControlFlowNode for x | runtime | int 7 | builtin-class int | +| b_condition.py:41 | ControlFlowNode for IntegerLiteral | import | int 1 | builtin-class int | +| b_condition.py:42 | ControlFlowNode for Compare | import | bool False | builtin-class bool | +| b_condition.py:42 | ControlFlowNode for Compare | import | bool True | builtin-class bool | +| b_condition.py:42 | ControlFlowNode for None | import | None | builtin-class NoneType | +| b_condition.py:43 | ControlFlowNode for Attribute | import | int 1 | builtin-class int | +| b_condition.py:50 | ControlFlowNode for FunctionExpr | import | Function g | builtin-class function | +| b_condition.py:55 | ControlFlowNode for FunctionExpr | import | Function loop | builtin-class function | +| b_condition.py:61 | ControlFlowNode for FunctionExpr | import | Function double_attr_check | builtin-class function | +| b_condition.py:62 | ControlFlowNode for Compare | runtime | bool False | builtin-class bool | +| b_condition.py:62 | ControlFlowNode for Compare | runtime | bool True | builtin-class bool | +| b_condition.py:62 | ControlFlowNode for IntegerLiteral | runtime | int 3 | builtin-class int | +| b_condition.py:65 | ControlFlowNode for Compare | runtime | bool False | builtin-class bool | +| b_condition.py:65 | ControlFlowNode for Compare | runtime | bool True | builtin-class bool | +| b_condition.py:65 | ControlFlowNode for IntegerLiteral | runtime | int 0 | builtin-class int | +| b_condition.py:66 | ControlFlowNode for Compare | runtime | bool False | builtin-class bool | +| b_condition.py:66 | ControlFlowNode for Compare | runtime | bool True | builtin-class bool | +| b_condition.py:69 | ControlFlowNode for FunctionExpr | import | Function h | builtin-class function | +| b_condition.py:70 | ControlFlowNode for IfExp | runtime | bool True | builtin-class bool | +| b_condition.py:70 | ControlFlowNode for True | runtime | bool True | builtin-class bool | +| b_condition.py:71 | ControlFlowNode for UnaryExpr | runtime | bool False | builtin-class bool | +| b_condition.py:71 | ControlFlowNode for UnaryExpr | runtime | bool True | builtin-class bool | +| b_condition.py:71 | ControlFlowNode for b | runtime | bool True | builtin-class bool | +| b_condition.py:72 | ControlFlowNode for IntegerLiteral | runtime | int 7 | builtin-class int | +| b_condition.py:73 | ControlFlowNode for b | runtime | bool True | builtin-class bool | +| b_condition.py:73 | ControlFlowNode for b | runtime | int 7 | builtin-class int | +| b_condition.py:75 | ControlFlowNode for FunctionExpr | import | Function k | builtin-class function | +| b_condition.py:76 | ControlFlowNode for type | runtime | builtin-class type | builtin-class type | +| b_condition.py:77 | ControlFlowNode for Compare | runtime | bool True | builtin-class bool | +| b_condition.py:77 | ControlFlowNode for object | runtime | builtin-class object | builtin-class type | +| b_condition.py:77 | ControlFlowNode for t | runtime | builtin-class type | builtin-class type | +| b_condition.py:78 | ControlFlowNode for object | runtime | builtin-class object | builtin-class type | +| b_condition.py:79 | ControlFlowNode for t | runtime | builtin-class object | builtin-class type | +| b_condition.py:81 | ControlFlowNode for FunctionExpr | import | Function odasa6261 | builtin-class function | +| b_condition.py:81 | ControlFlowNode for True | import | bool True | builtin-class bool | +| b_condition.py:82 | ControlFlowNode for callable | runtime | Builtin-function callable | builtin-class builtin_function_or_method | +| b_condition.py:82 | ControlFlowNode for callable() | runtime | bool False | builtin-class bool | +| b_condition.py:82 | ControlFlowNode for callable() | runtime | bool True | builtin-class bool | +| b_condition.py:82 | ControlFlowNode for foo | runtime | bool True | builtin-class bool | +| b_condition.py:83 | ControlFlowNode for FunctionExpr | runtime | Function odasa6261.bar | builtin-class function | +| b_condition.py:87 | ControlFlowNode for FunctionExpr | import | Function split_bool1 | builtin-class function | +| b_condition.py:87 | ControlFlowNode for None | import | None | builtin-class NoneType | +| b_condition.py:88 | ControlFlowNode for x | runtime | None | builtin-class NoneType | +| b_condition.py:88 | ControlFlowNode for y | runtime | None | builtin-class NoneType | +| b_condition.py:90 | ControlFlowNode for x | runtime | None | builtin-class NoneType | +| b_condition.py:90 | ControlFlowNode for y | runtime | None | builtin-class NoneType | +| b_condition.py:92 | ControlFlowNode for x | runtime | None | builtin-class NoneType | +| b_condition.py:93 | ControlFlowNode for y | runtime | None | builtin-class NoneType | +| b_condition.py:96 | ControlFlowNode for y | runtime | None | builtin-class NoneType | +| b_condition.py:97 | ControlFlowNode for x | runtime | None | builtin-class NoneType | +| b_condition.py:101 | ControlFlowNode for FunctionExpr | import | Function not_or_not | builtin-class function | +| b_condition.py:102 | ControlFlowNode for Tuple | runtime | (builtin-class tuple, builtin-class list) | builtin-class tuple | +| b_condition.py:102 | ControlFlowNode for a | runtime | instance of tuple | builtin-class tuple | +| b_condition.py:102 | ControlFlowNode for isinstance | runtime | Builtin-function isinstance | builtin-class builtin_function_or_method | +| b_condition.py:102 | ControlFlowNode for list | runtime | builtin-class list | builtin-class type | +| b_condition.py:102 | ControlFlowNode for tuple | runtime | builtin-class tuple | builtin-class type | +| e_temporal.py:2 | ControlFlowNode for ImportExpr | import | Module sys | builtin-class module | +| e_temporal.py:4 | ControlFlowNode for FunctionExpr | import | Function f | builtin-class function | +| e_temporal.py:5 | ControlFlowNode for Attribute | code/e_temporal.py:12 from import | list object | builtin-class list | +| e_temporal.py:5 | ControlFlowNode for Attribute | runtime | list object | builtin-class list | +| e_temporal.py:5 | ControlFlowNode for Compare | code/e_temporal.py:12 from import | bool False | builtin-class bool | +| e_temporal.py:5 | ControlFlowNode for Compare | code/e_temporal.py:12 from import | bool True | builtin-class bool | +| e_temporal.py:5 | ControlFlowNode for Compare | runtime | bool False | builtin-class bool | +| e_temporal.py:5 | ControlFlowNode for Compare | runtime | bool True | builtin-class bool | +| e_temporal.py:5 | ControlFlowNode for IntegerLiteral | code/e_temporal.py:12 from import | int 3 | builtin-class int | +| e_temporal.py:5 | ControlFlowNode for IntegerLiteral | runtime | int 3 | builtin-class int | +| e_temporal.py:5 | ControlFlowNode for len | code/e_temporal.py:12 from import | Builtin-function len | builtin-class builtin_function_or_method | +| e_temporal.py:5 | ControlFlowNode for len | runtime | Builtin-function len | builtin-class builtin_function_or_method | +| e_temporal.py:5 | ControlFlowNode for len() | code/e_temporal.py:12 from import | instance of int | builtin-class int | +| e_temporal.py:5 | ControlFlowNode for len() | runtime | instance of int | builtin-class int | +| e_temporal.py:5 | ControlFlowNode for sys | code/e_temporal.py:12 from import | Module sys | builtin-class module | +| e_temporal.py:5 | ControlFlowNode for sys | runtime | Module sys | builtin-class module | +| e_temporal.py:7 | ControlFlowNode for IntegerLiteral | code/e_temporal.py:12 from import | int 1 | builtin-class int | +| e_temporal.py:7 | ControlFlowNode for IntegerLiteral | runtime | int 1 | builtin-class int | +| e_temporal.py:9 | ControlFlowNode for FunctionExpr | import | Function g | builtin-class function | +| e_temporal.py:10 | ControlFlowNode for arg | code/e_temporal.py:12 from import | int 1 | builtin-class int | +| e_temporal.py:12 | ControlFlowNode for f | import | Function f | builtin-class function | +| e_temporal.py:12 | ControlFlowNode for f() | import | int 1 | builtin-class int | +| e_temporal.py:12 | ControlFlowNode for g | import | Function g | builtin-class function | +| e_temporal.py:12 | ControlFlowNode for g() | import | int 1 | builtin-class int | +| g_class_init.py:3 | ControlFlowNode for ClassExpr | import | class C | builtin-class type | +| g_class_init.py:3 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| g_class_init.py:5 | ControlFlowNode for FunctionExpr | import | Function C.__init__ | builtin-class function | +| g_class_init.py:6 | ControlFlowNode for Attribute | runtime | Method(Function C._init, self instance of C) | builtin-class method | +| g_class_init.py:6 | ControlFlowNode for Attribute() | runtime | None | builtin-class NoneType | +| g_class_init.py:6 | ControlFlowNode for self | runtime | self instance of C | class C | +| g_class_init.py:7 | ControlFlowNode for IntegerLiteral | runtime | int 1 | builtin-class int | +| g_class_init.py:7 | ControlFlowNode for self | runtime | self instance of C | class C | +| g_class_init.py:9 | ControlFlowNode for FunctionExpr | import | Function C._init | builtin-class function | +| g_class_init.py:10 | ControlFlowNode for IntegerLiteral | code/g_class_init.py:6 from runtime | int 2 | builtin-class int | +| g_class_init.py:10 | ControlFlowNode for self | code/g_class_init.py:6 from runtime | self instance of C | class C | +| g_class_init.py:11 | ControlFlowNode for Attribute | code/g_class_init.py:6 from runtime | Method(Function C._init2, self instance of C) | builtin-class method | +| g_class_init.py:11 | ControlFlowNode for Attribute() | code/g_class_init.py:6 from runtime | None | builtin-class NoneType | +| g_class_init.py:11 | ControlFlowNode for self | code/g_class_init.py:6 from runtime | self instance of C | class C | +| g_class_init.py:13 | ControlFlowNode for FunctionExpr | import | Function C._init2 | builtin-class function | +| g_class_init.py:14 | ControlFlowNode for IntegerLiteral | code/g_class_init.py:11 from code/g_class_init.py:6 from runtime | int 3 | builtin-class int | +| g_class_init.py:14 | ControlFlowNode for self | code/g_class_init.py:11 from code/g_class_init.py:6 from runtime | self instance of C | class C | +| g_class_init.py:16 | ControlFlowNode for FunctionExpr | import | Function C.method | builtin-class function | +| g_class_init.py:17 | ControlFlowNode for self | runtime | self instance of C | class C | +| g_class_init.py:18 | ControlFlowNode for int | runtime | builtin-class int | builtin-class type | +| g_class_init.py:18 | ControlFlowNode for isinstance | runtime | Builtin-function isinstance | builtin-class builtin_function_or_method | +| g_class_init.py:18 | ControlFlowNode for isinstance() | runtime | bool False | builtin-class bool | +| g_class_init.py:18 | ControlFlowNode for isinstance() | runtime | bool True | builtin-class bool | +| g_class_init.py:18 | ControlFlowNode for self | runtime | self instance of C | class C | +| g_class_init.py:19 | ControlFlowNode for self | runtime | self instance of C | class C | +| g_class_init.py:20 | ControlFlowNode for self | runtime | self instance of C | class C | +| g_class_init.py:24 | ControlFlowNode for ClassExpr | import | class Oddities | builtin-class type | +| g_class_init.py:24 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| g_class_init.py:26 | ControlFlowNode for int | import | builtin-class int | builtin-class type | +| g_class_init.py:27 | ControlFlowNode for float | import | builtin-class float | builtin-class type | +| g_class_init.py:28 | ControlFlowNode for len | import | Builtin-function len | builtin-class builtin_function_or_method | +| g_class_init.py:29 | ControlFlowNode for hash | import | Builtin-function hash | builtin-class builtin_function_or_method | +| g_class_init.py:32 | ControlFlowNode for ClassExpr | import | class D | builtin-class type | +| g_class_init.py:32 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| g_class_init.py:34 | ControlFlowNode for FunctionExpr | import | Function D.__init__ | builtin-class function | +| g_class_init.py:35 | ControlFlowNode for D | runtime | class D | builtin-class type | +| g_class_init.py:35 | ControlFlowNode for self | runtime | self instance of D | class D | +| g_class_init.py:35 | ControlFlowNode for super | runtime | builtin-class super | builtin-class type | +| g_class_init.py:35 | ControlFlowNode for super() | runtime | super(class D, self instance of D) | builtin-class super | +| g_class_init.py:36 | ControlFlowNode for Attribute | runtime | Method(builtin method __init__, self instance of D) | builtin-class method | +| g_class_init.py:36 | ControlFlowNode for D | runtime | class D | builtin-class type | +| g_class_init.py:36 | ControlFlowNode for self | runtime | self instance of D | class D | +| g_class_init.py:36 | ControlFlowNode for super | runtime | builtin-class super | builtin-class type | +| g_class_init.py:36 | ControlFlowNode for super() | runtime | super(class D, self instance of D) | builtin-class super | +| g_class_init.py:42 | ControlFlowNode for Str | import | 'v2' | builtin-class str | +| g_class_init.py:43 | ControlFlowNode for Str | import | 'v3' | builtin-class str | +| g_class_init.py:45 | ControlFlowNode for ClassExpr | import | class E | builtin-class type | +| g_class_init.py:45 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| g_class_init.py:46 | ControlFlowNode for FunctionExpr | import | Function E.__init__ | builtin-class function | +| g_class_init.py:48 | ControlFlowNode for V2 | runtime | 'v2' | builtin-class str | +| g_class_init.py:48 | ControlFlowNode for self | runtime | self instance of E | class E | +| g_class_init.py:50 | ControlFlowNode for V3 | runtime | 'v3' | builtin-class str | +| g_class_init.py:50 | ControlFlowNode for self | runtime | self instance of E | class E | +| g_class_init.py:52 | ControlFlowNode for FunctionExpr | import | Function E.meth | builtin-class function | +| g_class_init.py:53 | ControlFlowNode for Compare | runtime | bool False | builtin-class bool | +| g_class_init.py:53 | ControlFlowNode for Compare | runtime | bool True | builtin-class bool | +| g_class_init.py:53 | ControlFlowNode for V2 | runtime | 'v2' | builtin-class str | +| g_class_init.py:53 | ControlFlowNode for self | runtime | self instance of E | class E | +| h_classes.py:1 | ControlFlowNode for ImportExpr | import | Module sys | builtin-class module | +| h_classes.py:3 | ControlFlowNode for ClassExpr | import | class C | builtin-class type | +| h_classes.py:3 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| h_classes.py:5 | ControlFlowNode for Str | import | 'C_x' | builtin-class str | +| h_classes.py:7 | ControlFlowNode for FunctionExpr | import | Function C.__init__ | builtin-class function | +| h_classes.py:8 | ControlFlowNode for Str | code/h_classes.py:10 from import | 'c_y' | builtin-class str | +| h_classes.py:8 | ControlFlowNode for Str | code/h_classes.py:15 from runtime | 'c_y' | builtin-class str | +| h_classes.py:8 | ControlFlowNode for Str | runtime | 'c_y' | builtin-class str | +| h_classes.py:8 | ControlFlowNode for self | runtime | self instance of C | class C | +| h_classes.py:10 | ControlFlowNode for C | import | class C | builtin-class type | +| h_classes.py:10 | ControlFlowNode for C() | import | C() | class C | +| h_classes.py:10 | ControlFlowNode for type | import | builtin-class type | builtin-class type | +| h_classes.py:10 | ControlFlowNode for type() | import | class C | builtin-class type | +| h_classes.py:11 | ControlFlowNode for sys | import | Module sys | builtin-class module | +| h_classes.py:11 | ControlFlowNode for type | import | builtin-class type | builtin-class type | +| h_classes.py:11 | ControlFlowNode for type() | import | builtin-class module | builtin-class type | +| h_classes.py:12 | ControlFlowNode for Dict | import | Dict | builtin-class dict | +| h_classes.py:12 | ControlFlowNode for Tuple | import | (builtin-class object) | builtin-class tuple | +| h_classes.py:12 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| h_classes.py:12 | ControlFlowNode for type | import | builtin-class type | builtin-class type | +| h_classes.py:12 | ControlFlowNode for type() | import | type() | builtin-class type | +| h_classes.py:14 | ControlFlowNode for FunctionExpr | import | Function k | builtin-class function | +| h_classes.py:15 | ControlFlowNode for C | runtime | class C | builtin-class type | +| h_classes.py:15 | ControlFlowNode for C() | runtime | C() | class C | +| h_classes.py:15 | ControlFlowNode for type | runtime | builtin-class type | builtin-class type | +| h_classes.py:15 | ControlFlowNode for type() | runtime | class C | builtin-class type | +| h_classes.py:16 | ControlFlowNode for sys | runtime | Module sys | builtin-class module | +| h_classes.py:16 | ControlFlowNode for type | runtime | builtin-class type | builtin-class type | +| h_classes.py:16 | ControlFlowNode for type() | runtime | builtin-class module | builtin-class type | +| h_classes.py:17 | ControlFlowNode for type | runtime | builtin-class type | builtin-class type | +| h_classes.py:18 | ControlFlowNode for Dict | runtime | Dict | builtin-class dict | +| h_classes.py:18 | ControlFlowNode for Tuple | runtime | (builtin-class object) | builtin-class tuple | +| h_classes.py:18 | ControlFlowNode for object | runtime | builtin-class object | builtin-class type | +| h_classes.py:18 | ControlFlowNode for type | runtime | builtin-class type | builtin-class type | +| h_classes.py:18 | ControlFlowNode for type() | runtime | type() | builtin-class type | +| h_classes.py:23 | ControlFlowNode for ClassExpr | import | class Base | builtin-class type | +| h_classes.py:23 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| h_classes.py:25 | ControlFlowNode for FunctionExpr | import | Function Base.__init__ | builtin-class function | +| h_classes.py:26 | ControlFlowNode for Compare | code/h_classes.py:42 from import | bool False | builtin-class bool | +| h_classes.py:26 | ControlFlowNode for Compare | code/h_classes.py:42 from import | bool True | builtin-class bool | +| h_classes.py:26 | ControlFlowNode for Compare | runtime | bool False | builtin-class bool | +| h_classes.py:26 | ControlFlowNode for Compare | runtime | bool True | builtin-class bool | +| h_classes.py:26 | ControlFlowNode for IntegerLiteral | code/h_classes.py:42 from import | int 1 | builtin-class int | +| h_classes.py:26 | ControlFlowNode for IntegerLiteral | runtime | int 1 | builtin-class int | +| h_classes.py:27 | ControlFlowNode for Derived1 | code/h_classes.py:42 from import | class Derived1 | builtin-class type | +| h_classes.py:27 | ControlFlowNode for Derived1 | runtime | class Derived1 | builtin-class type | +| h_classes.py:27 | ControlFlowNode for self | runtime | self instance of Base | class Base | +| h_classes.py:28 | ControlFlowNode for Compare | code/h_classes.py:42 from import | bool False | builtin-class bool | +| h_classes.py:28 | ControlFlowNode for Compare | code/h_classes.py:42 from import | bool True | builtin-class bool | +| h_classes.py:28 | ControlFlowNode for Compare | runtime | bool False | builtin-class bool | +| h_classes.py:28 | ControlFlowNode for Compare | runtime | bool True | builtin-class bool | +| h_classes.py:28 | ControlFlowNode for IntegerLiteral | code/h_classes.py:42 from import | int 2 | builtin-class int | +| h_classes.py:28 | ControlFlowNode for IntegerLiteral | runtime | int 2 | builtin-class int | +| h_classes.py:29 | ControlFlowNode for Derived2 | code/h_classes.py:42 from import | class Derived2 | builtin-class type | +| h_classes.py:29 | ControlFlowNode for Derived2 | runtime | class Derived2 | builtin-class type | +| h_classes.py:29 | ControlFlowNode for self | runtime | self instance of Base | class Base | +| h_classes.py:31 | ControlFlowNode for Derived3 | code/h_classes.py:42 from import | class Derived3 | builtin-class type | +| h_classes.py:31 | ControlFlowNode for Derived3 | runtime | class Derived3 | builtin-class type | +| h_classes.py:31 | ControlFlowNode for self | runtime | self instance of Base | class Base | +| h_classes.py:33 | ControlFlowNode for Base | import | class Base | builtin-class type | +| h_classes.py:33 | ControlFlowNode for ClassExpr | import | class Derived1 | builtin-class type | +| h_classes.py:36 | ControlFlowNode for Base | import | class Base | builtin-class type | +| h_classes.py:36 | ControlFlowNode for ClassExpr | import | class Derived2 | builtin-class type | +| h_classes.py:39 | ControlFlowNode for Base | import | class Base | builtin-class type | +| h_classes.py:39 | ControlFlowNode for ClassExpr | import | class Derived3 | builtin-class type | +| h_classes.py:42 | ControlFlowNode for Base | import | class Base | builtin-class type | +| h_classes.py:45 | ControlFlowNode for FunctionExpr | import | Function f | builtin-class function | +| h_classes.py:48 | ControlFlowNode for ClassExpr | import | class D | builtin-class type | +| h_classes.py:48 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| h_classes.py:50 | ControlFlowNode for f | import | Function f | builtin-class function | +| h_classes.py:52 | ControlFlowNode for FunctionExpr | import | Function D.n | builtin-class function | +| i_imports.py:3 | ControlFlowNode for IntegerLiteral | import | int 1 | builtin-class int | +| i_imports.py:4 | ControlFlowNode for IntegerLiteral | import | int 2 | builtin-class int | +| i_imports.py:5 | ControlFlowNode for IntegerLiteral | import | int 3 | builtin-class int | +| i_imports.py:7 | ControlFlowNode for ImportExpr | import | Module code.xyz | builtin-class module | +| i_imports.py:8 | ControlFlowNode for ImportExpr | import | Package code | builtin-class module | +| i_imports.py:8 | ControlFlowNode for ImportMember | import | Module code.xyz | builtin-class module | +| i_imports.py:9 | ControlFlowNode for Attribute | import | float 1.0 | builtin-class float | +| i_imports.py:9 | ControlFlowNode for xyz | import | Module code.xyz | builtin-class module | +| i_imports.py:10 | ControlFlowNode for z | import | float 3.0 | builtin-class float | +| i_imports.py:11 | ControlFlowNode for a | import | int 1 | builtin-class int | +| i_imports.py:13 | ControlFlowNode for ImportExpr | import | Module sys | builtin-class module | +| i_imports.py:13 | ControlFlowNode for ImportMember | import | list object | builtin-class list | +| i_imports.py:15 | ControlFlowNode for argv | import | list object | builtin-class list | +| i_imports.py:17 | ControlFlowNode for ImportExpr | import | Module sys | builtin-class module | +| i_imports.py:18 | ControlFlowNode for Attribute | import | list object | builtin-class list | +| i_imports.py:18 | ControlFlowNode for sys | import | Module sys | builtin-class module | +| i_imports.py:23 | ControlFlowNode for ImportExpr | import | Package code | builtin-class module | +| i_imports.py:24 | ControlFlowNode for Attribute | import | Module code.package.x | builtin-class module | +| i_imports.py:24 | ControlFlowNode for Attribute | import | Package code.package | builtin-class module | +| i_imports.py:24 | ControlFlowNode for code | import | Package code | builtin-class module | +| i_imports.py:27 | ControlFlowNode for ImportExpr | import | Package code.test_package | builtin-class module | +| i_imports.py:29 | ControlFlowNode for ImportExpr | import | Module _io | builtin-class module | +| i_imports.py:30 | ControlFlowNode for Attribute | import | builtin-class _io.StringIO | builtin-class type | +| i_imports.py:30 | ControlFlowNode for _io | import | Module _io | builtin-class module | +| i_imports.py:31 | ControlFlowNode for Attribute | import | builtin-class _io.BytesIO | builtin-class type | +| i_imports.py:31 | ControlFlowNode for _io | import | Module _io | builtin-class module | +| i_imports.py:33 | ControlFlowNode for ImportExpr | import | Module io | builtin-class module | +| i_imports.py:34 | ControlFlowNode for Attribute | import | builtin-class _io.StringIO | builtin-class type | +| i_imports.py:34 | ControlFlowNode for io | import | Module io | builtin-class module | +| i_imports.py:35 | ControlFlowNode for Attribute | import | builtin-class _io.BytesIO | builtin-class type | +| i_imports.py:35 | ControlFlowNode for io | import | Module io | builtin-class module | +| i_imports.py:37 | ControlFlowNode for ImportExpr | import | Package code | builtin-class module | +| i_imports.py:38 | ControlFlowNode for Attribute | import | Function f2 | builtin-class function | +| i_imports.py:38 | ControlFlowNode for Attribute | import | Module code.n_nesting | builtin-class module | +| i_imports.py:38 | ControlFlowNode for Attribute() | import | None | builtin-class NoneType | +| i_imports.py:38 | ControlFlowNode for code | import | Package code | builtin-class module | +| j_convoluted_imports.py:2 | ControlFlowNode for ImportExpr | import | Package code.package | builtin-class module | +| j_convoluted_imports.py:3 | ControlFlowNode for ImportMember | import | Function module | builtin-class function | +| j_convoluted_imports.py:5 | ControlFlowNode for ImportExpr | import | Package code.package | builtin-class module | +| j_convoluted_imports.py:6 | ControlFlowNode for ImportMember | import | Module code.package.x | builtin-class module | +| j_convoluted_imports.py:9 | ControlFlowNode for ClassExpr | import | class C | builtin-class type | +| j_convoluted_imports.py:9 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| j_convoluted_imports.py:11 | ControlFlowNode for ImportExpr | import | Package code.package | builtin-class module | +| j_convoluted_imports.py:11 | ControlFlowNode for ImportMember | import | int 7 | builtin-class int | +| j_convoluted_imports.py:13 | ControlFlowNode for FunctionExpr | import | Function C.f | builtin-class function | +| j_convoluted_imports.py:14 | ControlFlowNode for ImportExpr | runtime | Package code.package | builtin-class module | +| j_convoluted_imports.py:14 | ControlFlowNode for ImportMember | runtime | Module code.package.x | builtin-class module | +| j_convoluted_imports.py:16 | ControlFlowNode for ImportExpr | import | Package code.package | builtin-class module | +| j_convoluted_imports.py:16 | ControlFlowNode for ImportMember | import | Module code.package.moduleX | builtin-class module | +| j_convoluted_imports.py:17 | ControlFlowNode for Attribute | import | class Y | builtin-class type | +| j_convoluted_imports.py:17 | ControlFlowNode for moduleX | import | Module code.package.moduleX | builtin-class module | +| k_getsetattr.py:4 | ControlFlowNode for ClassExpr | import | class C | builtin-class type | +| k_getsetattr.py:4 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| k_getsetattr.py:6 | ControlFlowNode for FunctionExpr | import | Function C.meth1 | builtin-class function | +| k_getsetattr.py:7 | ControlFlowNode for IntegerLiteral | code/k_getsetattr.py:15 from runtime | int 0 | builtin-class int | +| k_getsetattr.py:7 | ControlFlowNode for IntegerLiteral | runtime | int 0 | builtin-class int | +| k_getsetattr.py:7 | ControlFlowNode for Str | code/k_getsetattr.py:15 from runtime | 'a' | builtin-class str | +| k_getsetattr.py:7 | ControlFlowNode for Str | runtime | 'a' | builtin-class str | +| k_getsetattr.py:7 | ControlFlowNode for self | code/k_getsetattr.py:15 from runtime | self instance of C | class C | +| k_getsetattr.py:7 | ControlFlowNode for self | runtime | self instance of C | class C | +| k_getsetattr.py:7 | ControlFlowNode for setattr | code/k_getsetattr.py:15 from runtime | Builtin-function setattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:7 | ControlFlowNode for setattr | runtime | Builtin-function setattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:7 | ControlFlowNode for setattr() | code/k_getsetattr.py:15 from runtime | None | builtin-class NoneType | +| k_getsetattr.py:7 | ControlFlowNode for setattr() | runtime | None | builtin-class NoneType | +| k_getsetattr.py:8 | ControlFlowNode for IntegerLiteral | code/k_getsetattr.py:15 from runtime | int 1 | builtin-class int | +| k_getsetattr.py:8 | ControlFlowNode for IntegerLiteral | runtime | int 1 | builtin-class int | +| k_getsetattr.py:8 | ControlFlowNode for Str | code/k_getsetattr.py:15 from runtime | 'b' | builtin-class str | +| k_getsetattr.py:8 | ControlFlowNode for Str | runtime | 'b' | builtin-class str | +| k_getsetattr.py:8 | ControlFlowNode for self | code/k_getsetattr.py:15 from runtime | self instance of C | class C | +| k_getsetattr.py:8 | ControlFlowNode for self | runtime | self instance of C | class C | +| k_getsetattr.py:8 | ControlFlowNode for setattr | code/k_getsetattr.py:15 from runtime | Builtin-function setattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:8 | ControlFlowNode for setattr | runtime | Builtin-function setattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:8 | ControlFlowNode for setattr() | code/k_getsetattr.py:15 from runtime | None | builtin-class NoneType | +| k_getsetattr.py:8 | ControlFlowNode for setattr() | runtime | None | builtin-class NoneType | +| k_getsetattr.py:9 | ControlFlowNode for Str | code/k_getsetattr.py:15 from runtime | 'a' | builtin-class str | +| k_getsetattr.py:9 | ControlFlowNode for Str | runtime | 'a' | builtin-class str | +| k_getsetattr.py:9 | ControlFlowNode for getattr | code/k_getsetattr.py:15 from runtime | Builtin-function getattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:9 | ControlFlowNode for getattr | runtime | Builtin-function getattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:9 | ControlFlowNode for self | code/k_getsetattr.py:15 from runtime | self instance of C | class C | +| k_getsetattr.py:9 | ControlFlowNode for self | runtime | self instance of C | class C | +| k_getsetattr.py:10 | ControlFlowNode for Str | code/k_getsetattr.py:15 from runtime | 'c' | builtin-class str | +| k_getsetattr.py:10 | ControlFlowNode for Str | runtime | 'c' | builtin-class str | +| k_getsetattr.py:10 | ControlFlowNode for getattr | code/k_getsetattr.py:15 from runtime | Builtin-function getattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:10 | ControlFlowNode for getattr | runtime | Builtin-function getattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:10 | ControlFlowNode for self | code/k_getsetattr.py:15 from runtime | self instance of C | class C | +| k_getsetattr.py:10 | ControlFlowNode for self | runtime | self instance of C | class C | +| k_getsetattr.py:12 | ControlFlowNode for FunctionExpr | import | Function C.meth2 | builtin-class function | +| k_getsetattr.py:13 | ControlFlowNode for FloatLiteral | runtime | float 7.0 | builtin-class float | +| k_getsetattr.py:13 | ControlFlowNode for Str | runtime | 'a' | builtin-class str | +| k_getsetattr.py:13 | ControlFlowNode for self | runtime | self instance of C | class C | +| k_getsetattr.py:13 | ControlFlowNode for setattr | runtime | Builtin-function setattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:13 | ControlFlowNode for setattr() | runtime | None | builtin-class NoneType | +| k_getsetattr.py:14 | ControlFlowNode for IntegerLiteral | runtime | int 2 | builtin-class int | +| k_getsetattr.py:14 | ControlFlowNode for Str | runtime | 'c' | builtin-class str | +| k_getsetattr.py:14 | ControlFlowNode for self | runtime | self instance of C | class C | +| k_getsetattr.py:14 | ControlFlowNode for setattr | runtime | Builtin-function setattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:14 | ControlFlowNode for setattr() | runtime | None | builtin-class NoneType | +| k_getsetattr.py:15 | ControlFlowNode for Attribute | runtime | Method(Function C.meth1, self instance of C) | builtin-class method | +| k_getsetattr.py:15 | ControlFlowNode for Attribute() | runtime | None | builtin-class NoneType | +| k_getsetattr.py:15 | ControlFlowNode for self | runtime | self instance of C | class C | +| k_getsetattr.py:16 | ControlFlowNode for Str | runtime | 'a' | builtin-class str | +| k_getsetattr.py:16 | ControlFlowNode for getattr | runtime | Builtin-function getattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:16 | ControlFlowNode for self | runtime | self instance of C | class C | +| k_getsetattr.py:17 | ControlFlowNode for Str | runtime | 'b' | builtin-class str | +| k_getsetattr.py:17 | ControlFlowNode for getattr | runtime | Builtin-function getattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:17 | ControlFlowNode for self | runtime | self instance of C | class C | +| k_getsetattr.py:18 | ControlFlowNode for Str | runtime | 'c' | builtin-class str | +| k_getsetattr.py:18 | ControlFlowNode for getattr | runtime | Builtin-function getattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:18 | ControlFlowNode for self | runtime | self instance of C | class C | +| k_getsetattr.py:21 | ControlFlowNode for FunctionExpr | import | Function k | builtin-class function | +| k_getsetattr.py:22 | ControlFlowNode for C | runtime | class C | builtin-class type | +| k_getsetattr.py:22 | ControlFlowNode for C() | runtime | C() | class C | +| k_getsetattr.py:23 | ControlFlowNode for C | runtime | class C | builtin-class type | +| k_getsetattr.py:23 | ControlFlowNode for C() | runtime | C() | class C | +| k_getsetattr.py:24 | ControlFlowNode for C | runtime | class C | builtin-class type | +| k_getsetattr.py:24 | ControlFlowNode for C() | runtime | C() | class C | +| k_getsetattr.py:25 | ControlFlowNode for IntegerLiteral | runtime | int 10 | builtin-class int | +| k_getsetattr.py:25 | ControlFlowNode for c1 | runtime | C() | class C | +| k_getsetattr.py:27 | ControlFlowNode for IntegerLiteral | runtime | int 20 | builtin-class int | +| k_getsetattr.py:27 | ControlFlowNode for c2 | runtime | C() | class C | +| k_getsetattr.py:28 | ControlFlowNode for Attribute | runtime | int 10 | builtin-class int | +| k_getsetattr.py:28 | ControlFlowNode for c1 | runtime | C() | class C | +| k_getsetattr.py:29 | ControlFlowNode for Attribute | runtime | int 20 | builtin-class int | +| k_getsetattr.py:29 | ControlFlowNode for c2 | runtime | C() | class C | +| k_getsetattr.py:30 | ControlFlowNode for c3 | runtime | C() | class C | +| k_getsetattr.py:31 | ControlFlowNode for IntegerLiteral | runtime | int 30 | builtin-class int | +| k_getsetattr.py:31 | ControlFlowNode for c3 | runtime | C() | class C | +| l_calls.py:3 | ControlFlowNode for FunctionExpr | import | Function foo | builtin-class function | +| l_calls.py:3 | ControlFlowNode for List | import | List | builtin-class list | +| l_calls.py:4 | ControlFlowNode for Attribute | code/l_calls.py:9 from import | Method(builtin method append, List) | builtin-class method | +| l_calls.py:4 | ControlFlowNode for Attribute | runtime | Method(builtin method append, List) | builtin-class method | +| l_calls.py:4 | ControlFlowNode for Attribute() | code/l_calls.py:9 from import | None | builtin-class NoneType | +| l_calls.py:4 | ControlFlowNode for Attribute() | runtime | None | builtin-class NoneType | +| l_calls.py:4 | ControlFlowNode for Str | code/l_calls.py:9 from import | 'x' | builtin-class str | +| l_calls.py:4 | ControlFlowNode for Str | runtime | 'x' | builtin-class str | +| l_calls.py:4 | ControlFlowNode for x | code/l_calls.py:9 from import | List | builtin-class list | +| l_calls.py:4 | ControlFlowNode for x | runtime | List | builtin-class list | +| l_calls.py:6 | ControlFlowNode for FunctionExpr | import | Function bar | builtin-class function | +| l_calls.py:6 | ControlFlowNode for List | import | List | builtin-class list | +| l_calls.py:7 | ControlFlowNode for len | code/l_calls.py:10 from import | Builtin-function len | builtin-class builtin_function_or_method | +| l_calls.py:7 | ControlFlowNode for len | runtime | Builtin-function len | builtin-class builtin_function_or_method | +| l_calls.py:7 | ControlFlowNode for len() | code/l_calls.py:10 from import | instance of int | builtin-class int | +| l_calls.py:7 | ControlFlowNode for len() | runtime | instance of int | builtin-class int | +| l_calls.py:7 | ControlFlowNode for x | code/l_calls.py:10 from import | List | builtin-class list | +| l_calls.py:7 | ControlFlowNode for x | runtime | List | builtin-class list | +| l_calls.py:9 | ControlFlowNode for foo | import | Function foo | builtin-class function | +| l_calls.py:9 | ControlFlowNode for foo() | import | None | builtin-class NoneType | +| l_calls.py:10 | ControlFlowNode for bar | import | Function bar | builtin-class function | +| l_calls.py:10 | ControlFlowNode for bar() | import | instance of int | builtin-class int | +| l_calls.py:12 | ControlFlowNode for ClassExpr | import | class Owner | builtin-class type | +| l_calls.py:12 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| l_calls.py:14 | ControlFlowNode for classmethod | import | builtin-class classmethod | builtin-class type | +| l_calls.py:14 | ControlFlowNode for classmethod() | import | classmethod(Function Owner.cm) | builtin-class classmethod | +| l_calls.py:15 | ControlFlowNode for FunctionExpr | import | Function Owner.cm | builtin-class function | +| l_calls.py:16 | ControlFlowNode for cls | code/l_calls.py:24 from runtime | class Owner | builtin-class type | +| l_calls.py:18 | ControlFlowNode for classmethod | import | builtin-class classmethod | builtin-class type | +| l_calls.py:18 | ControlFlowNode for classmethod() | import | classmethod(Function Owner.cm2) | builtin-class classmethod | +| l_calls.py:19 | ControlFlowNode for FunctionExpr | import | Function Owner.cm2 | builtin-class function | +| l_calls.py:20 | ControlFlowNode for arg | code/l_calls.py:25 from runtime | int 1 | builtin-class int | +| l_calls.py:23 | ControlFlowNode for FunctionExpr | import | Function Owner.m | builtin-class function | +| l_calls.py:24 | ControlFlowNode for Attribute | runtime | Method(Function Owner.cm, class Owner) | builtin-class method | +| l_calls.py:24 | ControlFlowNode for Attribute() | runtime | class Owner | builtin-class type | +| l_calls.py:24 | ControlFlowNode for IntegerLiteral | runtime | int 0 | builtin-class int | +| l_calls.py:24 | ControlFlowNode for self | runtime | self instance of Owner | class Owner | +| l_calls.py:25 | ControlFlowNode for Attribute | runtime | Method(Function Owner.cm2, class Owner) | builtin-class method | +| l_calls.py:25 | ControlFlowNode for Attribute() | runtime | int 1 | builtin-class int | +| l_calls.py:25 | ControlFlowNode for IntegerLiteral | runtime | int 1 | builtin-class int | +| l_calls.py:25 | ControlFlowNode for a | runtime | class Owner | builtin-class type | +| m_attributes.py:3 | ControlFlowNode for ClassExpr | import | class C | builtin-class type | +| m_attributes.py:3 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| m_attributes.py:5 | ControlFlowNode for FunctionExpr | import | Function C.__init__ | builtin-class function | +| m_attributes.py:5 | ControlFlowNode for IntegerLiteral | import | int 17 | builtin-class int | +| m_attributes.py:6 | ControlFlowNode for a | code/m_attributes.py:13 from import | int 100 | builtin-class int | +| m_attributes.py:6 | ControlFlowNode for a | runtime | int 17 | builtin-class int | +| m_attributes.py:6 | ControlFlowNode for self | runtime | self instance of C | class C | +| m_attributes.py:8 | ControlFlowNode for FunctionExpr | import | Function C.foo | builtin-class function | +| m_attributes.py:9 | ControlFlowNode for self | code/m_attributes.py:12 from import | C() | class C | +| m_attributes.py:9 | ControlFlowNode for self | code/m_attributes.py:13 from import | C() | class C | +| m_attributes.py:9 | ControlFlowNode for self | runtime | self instance of C | class C | +| m_attributes.py:10 | ControlFlowNode for Attribute | code/m_attributes.py:13 from import | int 100 | builtin-class int | +| m_attributes.py:10 | ControlFlowNode for other | code/m_attributes.py:12 from import | C() | class C | +| m_attributes.py:10 | ControlFlowNode for other | code/m_attributes.py:13 from import | C() | class C | +| m_attributes.py:12 | ControlFlowNode for Attribute | import | Method(Function C.foo, C()) | builtin-class method | +| m_attributes.py:12 | ControlFlowNode for Attribute() | import | None | builtin-class NoneType | +| m_attributes.py:12 | ControlFlowNode for C | import | class C | builtin-class type | +| m_attributes.py:12 | ControlFlowNode for C() | import | C() | class C | +| m_attributes.py:12 | ControlFlowNode for C() | import | C() | class C | +| m_attributes.py:13 | ControlFlowNode for Attribute | import | Method(Function C.foo, C()) | builtin-class method | +| m_attributes.py:13 | ControlFlowNode for Attribute() | import | None | builtin-class NoneType | +| m_attributes.py:13 | ControlFlowNode for C | import | class C | builtin-class type | +| m_attributes.py:13 | ControlFlowNode for C() | import | C() | class C | +| m_attributes.py:13 | ControlFlowNode for C() | import | C() | class C | +| m_attributes.py:13 | ControlFlowNode for IntegerLiteral | import | int 100 | builtin-class int | +| n_nesting.py:8 | ControlFlowNode for FunctionExpr | import | Function foo | builtin-class function | +| n_nesting.py:8 | ControlFlowNode for True | import | bool True | builtin-class bool | +| n_nesting.py:9 | ControlFlowNode for callable | runtime | Builtin-function callable | builtin-class builtin_function_or_method | +| n_nesting.py:9 | ControlFlowNode for callable() | runtime | bool False | builtin-class bool | +| n_nesting.py:9 | ControlFlowNode for callable() | runtime | bool True | builtin-class bool | +| n_nesting.py:9 | ControlFlowNode for compile_ops | runtime | bool True | builtin-class bool | +| n_nesting.py:10 | ControlFlowNode for FunctionExpr | runtime | Function foo.inner | builtin-class function | +| n_nesting.py:13 | ControlFlowNode for FunctionExpr | runtime | Function foo.inner | builtin-class function | +| n_nesting.py:15 | ControlFlowNode for Dict | runtime | Dict | builtin-class dict | +| n_nesting.py:16 | ControlFlowNode for Str | runtime | 'inner' | builtin-class str | +| n_nesting.py:16 | ControlFlowNode for inner | runtime | Function foo.inner | builtin-class function | +| n_nesting.py:16 | ControlFlowNode for inner | runtime | Function foo.inner | builtin-class function | +| n_nesting.py:18 | ControlFlowNode for attrs | runtime | Dict | builtin-class dict | +| n_nesting.py:22 | ControlFlowNode for FunctionExpr | import | Function f1 | builtin-class function | +| n_nesting.py:23 | ControlFlowNode for C | code/n_nesting.py:25 from code/i_imports.py:38 from import | int 1 | builtin-class int | +| n_nesting.py:23 | ControlFlowNode for C | code/n_nesting.py:25 from code/n_nesting.py:27 from code/n_nesting.py:29 from runtime | int 1 | builtin-class int | +| n_nesting.py:23 | ControlFlowNode for C | code/n_nesting.py:25 from code/n_nesting.py:27 from runtime | int 1 | builtin-class int | +| n_nesting.py:23 | ControlFlowNode for C | code/n_nesting.py:25 from runtime | int 1 | builtin-class int | +| n_nesting.py:23 | ControlFlowNode for C | runtime | int 1 | builtin-class int | +| n_nesting.py:23 | ControlFlowNode for IntegerLiteral | code/n_nesting.py:25 from code/i_imports.py:38 from import | int 1 | builtin-class int | +| n_nesting.py:23 | ControlFlowNode for IntegerLiteral | code/n_nesting.py:25 from code/n_nesting.py:27 from code/n_nesting.py:29 from runtime | int 1 | builtin-class int | +| n_nesting.py:23 | ControlFlowNode for IntegerLiteral | code/n_nesting.py:25 from code/n_nesting.py:27 from runtime | int 1 | builtin-class int | +| n_nesting.py:23 | ControlFlowNode for IntegerLiteral | code/n_nesting.py:25 from runtime | int 1 | builtin-class int | +| n_nesting.py:23 | ControlFlowNode for IntegerLiteral | runtime | int 1 | builtin-class int | +| n_nesting.py:24 | ControlFlowNode for FunctionExpr | import | Function f2 | builtin-class function | +| n_nesting.py:25 | ControlFlowNode for f1 | code/i_imports.py:38 from import | Function f1 | builtin-class function | +| n_nesting.py:25 | ControlFlowNode for f1 | code/n_nesting.py:27 from code/n_nesting.py:29 from code/n_nesting.py:31 from import | Function f1 | builtin-class function | +| n_nesting.py:25 | ControlFlowNode for f1 | code/n_nesting.py:27 from code/n_nesting.py:29 from runtime | Function f1 | builtin-class function | +| n_nesting.py:25 | ControlFlowNode for f1 | code/n_nesting.py:27 from runtime | Function f1 | builtin-class function | +| n_nesting.py:25 | ControlFlowNode for f1 | runtime | Function f1 | builtin-class function | +| n_nesting.py:25 | ControlFlowNode for f1() | code/i_imports.py:38 from import | None | builtin-class NoneType | +| n_nesting.py:25 | ControlFlowNode for f1() | code/n_nesting.py:27 from code/n_nesting.py:29 from code/n_nesting.py:31 from import | None | builtin-class NoneType | +| n_nesting.py:25 | ControlFlowNode for f1() | code/n_nesting.py:27 from code/n_nesting.py:29 from runtime | None | builtin-class NoneType | +| n_nesting.py:25 | ControlFlowNode for f1() | code/n_nesting.py:27 from runtime | None | builtin-class NoneType | +| n_nesting.py:25 | ControlFlowNode for f1() | runtime | None | builtin-class NoneType | +| n_nesting.py:26 | ControlFlowNode for FunctionExpr | import | Function f3 | builtin-class function | +| n_nesting.py:27 | ControlFlowNode for f2 | code/n_nesting.py:29 from code/n_nesting.py:31 from import | Function f2 | builtin-class function | +| n_nesting.py:27 | ControlFlowNode for f2 | code/n_nesting.py:29 from runtime | Function f2 | builtin-class function | +| n_nesting.py:27 | ControlFlowNode for f2 | runtime | Function f2 | builtin-class function | +| n_nesting.py:27 | ControlFlowNode for f2() | code/n_nesting.py:29 from code/n_nesting.py:31 from import | None | builtin-class NoneType | +| n_nesting.py:27 | ControlFlowNode for f2() | code/n_nesting.py:29 from runtime | None | builtin-class NoneType | +| n_nesting.py:27 | ControlFlowNode for f2() | runtime | None | builtin-class NoneType | +| n_nesting.py:28 | ControlFlowNode for FunctionExpr | import | Function f4 | builtin-class function | +| n_nesting.py:29 | ControlFlowNode for f3 | code/n_nesting.py:31 from import | Function f3 | builtin-class function | +| n_nesting.py:29 | ControlFlowNode for f3 | runtime | Function f3 | builtin-class function | +| n_nesting.py:29 | ControlFlowNode for f3() | code/n_nesting.py:31 from import | None | builtin-class NoneType | +| n_nesting.py:29 | ControlFlowNode for f3() | runtime | None | builtin-class NoneType | +| n_nesting.py:30 | ControlFlowNode for ClassExpr | import | class C | builtin-class type | +| n_nesting.py:30 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| n_nesting.py:31 | ControlFlowNode for f4 | import | Function f4 | builtin-class function | +| n_nesting.py:31 | ControlFlowNode for f4() | import | None | builtin-class NoneType | +| n_nesting.py:32 | ControlFlowNode for C | import | class C | builtin-class type | +| n_nesting.py:32 | ControlFlowNode for ClassExpr | import | class D | builtin-class type | +| n_nesting.py:34 | ControlFlowNode for IntegerLiteral | import | int 1 | builtin-class int | +| p_decorators.py:3 | ControlFlowNode for FunctionExpr | import | Function simple | builtin-class function | +| p_decorators.py:4 | ControlFlowNode for Str | code/p_decorators.py:7 from import | 'Hello' | builtin-class str | +| p_decorators.py:4 | ControlFlowNode for Str | runtime | 'Hello' | builtin-class str | +| p_decorators.py:4 | ControlFlowNode for func | code/p_decorators.py:7 from import | Function foo | builtin-class function | +| p_decorators.py:5 | ControlFlowNode for func | code/p_decorators.py:7 from import | Function foo | builtin-class function | +| p_decorators.py:7 | ControlFlowNode for simple | import | Function simple | builtin-class function | +| p_decorators.py:7 | ControlFlowNode for simple() | import | Function foo | builtin-class function | +| p_decorators.py:8 | ControlFlowNode for FunctionExpr | import | Function foo | builtin-class function | +| p_decorators.py:11 | ControlFlowNode for FunctionExpr | import | Function complex | builtin-class function | +| p_decorators.py:12 | ControlFlowNode for FunctionExpr | code/p_decorators.py:17 from import | Function complex.annotate | builtin-class function | +| p_decorators.py:12 | ControlFlowNode for FunctionExpr | runtime | Function complex.annotate | builtin-class function | +| p_decorators.py:13 | ControlFlowNode for func | code/p_decorators.py:17 from import | Function bar | builtin-class function | +| p_decorators.py:14 | ControlFlowNode for func | code/p_decorators.py:17 from import | Function bar | builtin-class function | +| p_decorators.py:15 | ControlFlowNode for annotate | code/p_decorators.py:17 from import | Function complex.annotate | builtin-class function | +| p_decorators.py:15 | ControlFlowNode for annotate | runtime | Function complex.annotate | builtin-class function | +| p_decorators.py:17 | ControlFlowNode for Str | import | 'Hi' | builtin-class str | +| p_decorators.py:17 | ControlFlowNode for complex | import | Function complex | builtin-class function | +| p_decorators.py:17 | ControlFlowNode for complex() | import | Function complex.annotate | builtin-class function | +| p_decorators.py:17 | ControlFlowNode for complex()() | import | Function bar | builtin-class function | +| p_decorators.py:18 | ControlFlowNode for FunctionExpr | import | Function bar | builtin-class function | +| p_decorators.py:21 | ControlFlowNode for foo | import | Function foo | builtin-class function | +| p_decorators.py:22 | ControlFlowNode for bar | import | Function bar | builtin-class function | +| p_decorators.py:24 | ControlFlowNode for ClassExpr | import | class C | builtin-class type | +| p_decorators.py:24 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| p_decorators.py:26 | ControlFlowNode for staticmethod | import | builtin-class staticmethod | builtin-class type | +| p_decorators.py:26 | ControlFlowNode for staticmethod() | import | staticmethod() | builtin-class staticmethod | +| p_decorators.py:27 | ControlFlowNode for FunctionExpr | import | Function C.smeth | builtin-class function | +| p_decorators.py:31 | ControlFlowNode for classmethod | import | builtin-class classmethod | builtin-class type | +| p_decorators.py:31 | ControlFlowNode for classmethod() | import | classmethod(Function C.cmeth) | builtin-class classmethod | +| p_decorators.py:32 | ControlFlowNode for FunctionExpr | import | Function C.cmeth | builtin-class function | +| q_super.py:1 | ControlFlowNode for ClassExpr | import | class Base2 | builtin-class type | +| q_super.py:1 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| q_super.py:3 | ControlFlowNode for FunctionExpr | import | Function Base2.__init__ | builtin-class function | +| q_super.py:4 | ControlFlowNode for Attribute | code/q_super.py:12 from runtime | Method(builtin method __init__, self instance of Derived4) | builtin-class method | +| q_super.py:4 | ControlFlowNode for Attribute | runtime | Method(builtin method __init__, self instance of Base2) | builtin-class method | +| q_super.py:4 | ControlFlowNode for Base2 | code/q_super.py:12 from runtime | class Base2 | builtin-class type | +| q_super.py:4 | ControlFlowNode for Base2 | runtime | class Base2 | builtin-class type | +| q_super.py:4 | ControlFlowNode for self | code/q_super.py:12 from runtime | self instance of Derived4 | class Derived4 | +| q_super.py:4 | ControlFlowNode for self | runtime | self instance of Base2 | class Base2 | +| q_super.py:4 | ControlFlowNode for super | code/q_super.py:12 from runtime | builtin-class super | builtin-class type | +| q_super.py:4 | ControlFlowNode for super | runtime | builtin-class super | builtin-class type | +| q_super.py:4 | ControlFlowNode for super() | code/q_super.py:12 from runtime | super(class Base2, self instance of Derived4) | builtin-class super | +| q_super.py:4 | ControlFlowNode for super() | runtime | super(class Base2, self instance of Base2) | builtin-class super | +| q_super.py:8 | ControlFlowNode for Base2 | import | class Base2 | builtin-class type | +| q_super.py:8 | ControlFlowNode for ClassExpr | import | class Derived4 | builtin-class type | +| q_super.py:10 | ControlFlowNode for FunctionExpr | import | Function Derived4.__init__ | builtin-class function | +| q_super.py:11 | ControlFlowNode for Base2 | runtime | class Base2 | builtin-class type | +| q_super.py:11 | ControlFlowNode for self | runtime | self instance of Derived4 | class Derived4 | +| q_super.py:11 | ControlFlowNode for super | runtime | builtin-class super | builtin-class type | +| q_super.py:11 | ControlFlowNode for super() | runtime | super(class Base2, self instance of Derived4) | builtin-class super | +| q_super.py:12 | ControlFlowNode for Attribute | runtime | Method(Function Base2.__init__, self instance of Derived4) | builtin-class method | +| q_super.py:12 | ControlFlowNode for Attribute() | runtime | None | builtin-class NoneType | +| q_super.py:12 | ControlFlowNode for Derived4 | runtime | class Derived4 | builtin-class type | +| q_super.py:12 | ControlFlowNode for self | runtime | self instance of Derived4 | class Derived4 | +| q_super.py:12 | ControlFlowNode for super | runtime | builtin-class super | builtin-class type | +| q_super.py:12 | ControlFlowNode for super() | runtime | super(class Derived4, self instance of Derived4) | builtin-class super | +| q_super.py:14 | ControlFlowNode for ClassExpr | import | class Base1 | builtin-class type | +| q_super.py:14 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| q_super.py:16 | ControlFlowNode for FunctionExpr | import | Function Base1.meth | builtin-class function | +| q_super.py:17 | ControlFlowNode for IntegerLiteral | code/q_super.py:22 from code/q_super.py:27 from code/q_super.py:38 from runtime | int 7 | builtin-class int | +| q_super.py:17 | ControlFlowNode for IntegerLiteral | code/q_super.py:22 from code/q_super.py:27 from runtime | int 7 | builtin-class int | +| q_super.py:17 | ControlFlowNode for IntegerLiteral | code/q_super.py:22 from code/q_super.py:32 from runtime | int 7 | builtin-class int | +| q_super.py:17 | ControlFlowNode for IntegerLiteral | code/q_super.py:22 from runtime | int 7 | builtin-class int | +| q_super.py:17 | ControlFlowNode for IntegerLiteral | runtime | int 7 | builtin-class int | +| q_super.py:19 | ControlFlowNode for Base1 | import | class Base1 | builtin-class type | +| q_super.py:19 | ControlFlowNode for ClassExpr | import | class Derived1 | builtin-class type | +| q_super.py:21 | ControlFlowNode for FunctionExpr | import | Function Derived1.meth | builtin-class function | +| q_super.py:22 | ControlFlowNode for Attribute | code/q_super.py:27 from code/q_super.py:38 from runtime | Method(Function Base1.meth, self instance of Wrong1) | builtin-class method | +| q_super.py:22 | ControlFlowNode for Attribute | code/q_super.py:27 from runtime | Method(Function Base1.meth, self instance of Derived2) | builtin-class method | +| q_super.py:22 | ControlFlowNode for Attribute | code/q_super.py:32 from runtime | Method(Function Base1.meth, self instance of Derived5) | builtin-class method | +| q_super.py:22 | ControlFlowNode for Attribute | runtime | Method(Function Base1.meth, self instance of Derived1) | builtin-class method | +| q_super.py:22 | ControlFlowNode for Attribute() | code/q_super.py:27 from code/q_super.py:38 from runtime | int 7 | builtin-class int | +| q_super.py:22 | ControlFlowNode for Attribute() | code/q_super.py:27 from runtime | int 7 | builtin-class int | +| q_super.py:22 | ControlFlowNode for Attribute() | code/q_super.py:32 from runtime | int 7 | builtin-class int | +| q_super.py:22 | ControlFlowNode for Attribute() | runtime | int 7 | builtin-class int | +| q_super.py:22 | ControlFlowNode for Derived1 | code/q_super.py:27 from code/q_super.py:38 from runtime | class Derived1 | builtin-class type | +| q_super.py:22 | ControlFlowNode for Derived1 | code/q_super.py:27 from runtime | class Derived1 | builtin-class type | +| q_super.py:22 | ControlFlowNode for Derived1 | code/q_super.py:32 from runtime | class Derived1 | builtin-class type | +| q_super.py:22 | ControlFlowNode for Derived1 | runtime | class Derived1 | builtin-class type | +| q_super.py:22 | ControlFlowNode for self | code/q_super.py:27 from code/q_super.py:38 from runtime | self instance of Wrong1 | class Wrong1 | +| q_super.py:22 | ControlFlowNode for self | code/q_super.py:27 from runtime | self instance of Derived2 | class Derived2 | +| q_super.py:22 | ControlFlowNode for self | code/q_super.py:32 from runtime | self instance of Derived5 | class Derived5 | +| q_super.py:22 | ControlFlowNode for self | runtime | self instance of Derived1 | class Derived1 | +| q_super.py:22 | ControlFlowNode for super | code/q_super.py:27 from code/q_super.py:38 from runtime | builtin-class super | builtin-class type | +| q_super.py:22 | ControlFlowNode for super | code/q_super.py:27 from runtime | builtin-class super | builtin-class type | +| q_super.py:22 | ControlFlowNode for super | code/q_super.py:32 from runtime | builtin-class super | builtin-class type | +| q_super.py:22 | ControlFlowNode for super | runtime | builtin-class super | builtin-class type | +| q_super.py:22 | ControlFlowNode for super() | code/q_super.py:27 from code/q_super.py:38 from runtime | super(class Derived1, self instance of Wrong1) | builtin-class super | +| q_super.py:22 | ControlFlowNode for super() | code/q_super.py:27 from runtime | super(class Derived1, self instance of Derived2) | builtin-class super | +| q_super.py:22 | ControlFlowNode for super() | code/q_super.py:32 from runtime | super(class Derived1, self instance of Derived5) | builtin-class super | +| q_super.py:22 | ControlFlowNode for super() | runtime | super(class Derived1, self instance of Derived1) | builtin-class super | +| q_super.py:24 | ControlFlowNode for ClassExpr | import | class Derived2 | builtin-class type | +| q_super.py:24 | ControlFlowNode for Derived1 | import | class Derived1 | builtin-class type | +| q_super.py:26 | ControlFlowNode for FunctionExpr | import | Function Derived2.meth | builtin-class function | +| q_super.py:27 | ControlFlowNode for Attribute | code/q_super.py:38 from runtime | Method(Function Derived1.meth, self instance of Wrong1) | builtin-class method | +| q_super.py:27 | ControlFlowNode for Attribute | runtime | Method(Function Derived1.meth, self instance of Derived2) | builtin-class method | +| q_super.py:27 | ControlFlowNode for Attribute() | code/q_super.py:38 from runtime | int 7 | builtin-class int | +| q_super.py:27 | ControlFlowNode for Attribute() | runtime | int 7 | builtin-class int | +| q_super.py:27 | ControlFlowNode for Derived2 | code/q_super.py:38 from runtime | class Derived2 | builtin-class type | +| q_super.py:27 | ControlFlowNode for Derived2 | runtime | class Derived2 | builtin-class type | +| q_super.py:27 | ControlFlowNode for self | code/q_super.py:38 from runtime | self instance of Wrong1 | class Wrong1 | +| q_super.py:27 | ControlFlowNode for self | runtime | self instance of Derived2 | class Derived2 | +| q_super.py:27 | ControlFlowNode for super | code/q_super.py:38 from runtime | builtin-class super | builtin-class type | +| q_super.py:27 | ControlFlowNode for super | runtime | builtin-class super | builtin-class type | +| q_super.py:27 | ControlFlowNode for super() | code/q_super.py:38 from runtime | super(class Derived2, self instance of Wrong1) | builtin-class super | +| q_super.py:27 | ControlFlowNode for super() | runtime | super(class Derived2, self instance of Derived2) | builtin-class super | +| q_super.py:29 | ControlFlowNode for ClassExpr | import | class Derived5 | builtin-class type | +| q_super.py:29 | ControlFlowNode for Derived1 | import | class Derived1 | builtin-class type | +| q_super.py:31 | ControlFlowNode for FunctionExpr | import | Function Derived5.meth | builtin-class function | +| q_super.py:32 | ControlFlowNode for Attribute | runtime | Method(Function Derived1.meth, self instance of Derived5) | builtin-class method | +| q_super.py:32 | ControlFlowNode for Attribute() | runtime | int 7 | builtin-class int | +| q_super.py:32 | ControlFlowNode for Derived5 | runtime | class Derived5 | builtin-class type | +| q_super.py:32 | ControlFlowNode for self | runtime | self instance of Derived5 | class Derived5 | +| q_super.py:32 | ControlFlowNode for super | runtime | builtin-class super | builtin-class type | +| q_super.py:32 | ControlFlowNode for super() | runtime | super(class Derived5, self instance of Derived5) | builtin-class super | +| q_super.py:35 | ControlFlowNode for ClassExpr | import | class Wrong1 | builtin-class type | +| q_super.py:35 | ControlFlowNode for Derived2 | import | class Derived2 | builtin-class type | +| q_super.py:35 | ControlFlowNode for Derived5 | import | class Derived5 | builtin-class type | +| q_super.py:37 | ControlFlowNode for FunctionExpr | import | Function Wrong1.meth | builtin-class function | +| q_super.py:38 | ControlFlowNode for Attribute | runtime | Method(Function Derived2.meth, self instance of Wrong1) | builtin-class method | +| q_super.py:38 | ControlFlowNode for Attribute() | runtime | int 7 | builtin-class int | +| q_super.py:38 | ControlFlowNode for Derived5 | runtime | class Derived5 | builtin-class type | +| q_super.py:38 | ControlFlowNode for self | runtime | self instance of Wrong1 | class Wrong1 | +| q_super.py:38 | ControlFlowNode for super | runtime | builtin-class super | builtin-class type | +| q_super.py:38 | ControlFlowNode for super() | runtime | super(class Derived5, self instance of Wrong1) | builtin-class super | +| q_super.py:41 | ControlFlowNode for ClassExpr | import | class DA | builtin-class type | +| q_super.py:41 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| q_super.py:43 | ControlFlowNode for FunctionExpr | import | Function DA.__init__ | builtin-class function | +| q_super.py:46 | ControlFlowNode for ClassExpr | import | class DB | builtin-class type | +| q_super.py:46 | ControlFlowNode for DA | import | class DA | builtin-class type | +| q_super.py:48 | ControlFlowNode for ClassExpr | import | class DC | builtin-class type | +| q_super.py:48 | ControlFlowNode for DA | import | class DA | builtin-class type | +| q_super.py:50 | ControlFlowNode for FunctionExpr | import | Function DB.DC.__init__ | builtin-class function | +| q_super.py:51 | ControlFlowNode for Attribute | runtime | class DC | builtin-class type | +| q_super.py:51 | ControlFlowNode for DB | runtime | class DB | builtin-class type | +| q_super.py:51 | ControlFlowNode for self | runtime | self instance of DC | class DC | +| q_super.py:51 | ControlFlowNode for super | runtime | builtin-class super | builtin-class type | +| q_super.py:51 | ControlFlowNode for super() | runtime | super(class DC, self instance of DC) | builtin-class super | +| q_super.py:52 | ControlFlowNode for Attribute | runtime | Method(Function DA.__init__, self instance of DC) | builtin-class method | +| q_super.py:52 | ControlFlowNode for Attribute() | runtime | None | builtin-class NoneType | +| q_super.py:52 | ControlFlowNode for sup | runtime | super(class DC, self instance of DC) | builtin-class super | +| q_super.py:55 | ControlFlowNode for ClassExpr | import | class DD | builtin-class type | +| q_super.py:55 | ControlFlowNode for DA | import | class DA | builtin-class type | +| q_super.py:57 | ControlFlowNode for FunctionExpr | import | Function DD.__init__ | builtin-class function | +| q_super.py:58 | ControlFlowNode for DD | runtime | class DD | builtin-class type | +| q_super.py:58 | ControlFlowNode for self | runtime | self instance of DD | class DD | +| q_super.py:58 | ControlFlowNode for super | runtime | builtin-class super | builtin-class type | +| q_super.py:58 | ControlFlowNode for super() | runtime | super(class DD, self instance of DD) | builtin-class super | +| q_super.py:59 | ControlFlowNode for Attribute | runtime | Method(Function DA.__init__, self instance of DD) | builtin-class method | +| q_super.py:59 | ControlFlowNode for Attribute() | runtime | None | builtin-class NoneType | +| q_super.py:59 | ControlFlowNode for sup | runtime | super(class DD, self instance of DD) | builtin-class super | +| q_super.py:61 | ControlFlowNode for ClassExpr | import | class DE | builtin-class type | +| q_super.py:61 | ControlFlowNode for DA | import | class DA | builtin-class type | +| q_super.py:63 | ControlFlowNode for ClassExpr | import | class DF | builtin-class type | +| q_super.py:63 | ControlFlowNode for DA | import | class DA | builtin-class type | +| q_super.py:65 | ControlFlowNode for FunctionExpr | import | Function DE.DF.__init__ | builtin-class function | +| q_super.py:66 | ControlFlowNode for Attribute | runtime | Method(Function DA.__init__, self instance of DF) | builtin-class method | +| q_super.py:66 | ControlFlowNode for Attribute | runtime | class DF | builtin-class type | +| q_super.py:66 | ControlFlowNode for Attribute() | runtime | None | builtin-class NoneType | +| q_super.py:66 | ControlFlowNode for DE | runtime | class DE | builtin-class type | +| q_super.py:66 | ControlFlowNode for self | runtime | self instance of DF | class DF | +| q_super.py:66 | ControlFlowNode for super | runtime | builtin-class super | builtin-class type | +| q_super.py:66 | ControlFlowNode for super() | runtime | super(class DF, self instance of DF) | builtin-class super | +| q_super.py:68 | ControlFlowNode for ClassExpr | import | class N | builtin-class type | +| q_super.py:68 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| q_super.py:71 | ControlFlowNode for ClassExpr | import | class M | builtin-class type | +| q_super.py:71 | ControlFlowNode for N | import | class N | builtin-class type | +| q_super.py:73 | ControlFlowNode for FunctionExpr | import | Function M.__init__ | builtin-class function | +| q_super.py:74 | ControlFlowNode for M | runtime | class M | builtin-class type | +| q_super.py:74 | ControlFlowNode for self | runtime | self instance of M | class M | +| q_super.py:74 | ControlFlowNode for super | runtime | builtin-class super | builtin-class type | +| q_super.py:74 | ControlFlowNode for super() | runtime | super(class M, self instance of M) | builtin-class super | +| q_super.py:75 | ControlFlowNode for Attribute | runtime | Method(builtin method __init__, self instance of M) | builtin-class method | +| q_super.py:75 | ControlFlowNode for s | runtime | super(class M, self instance of M) | builtin-class super | +| q_super.py:76 | ControlFlowNode for i | runtime | Method(builtin method __init__, self instance of M) | builtin-class method | +| r_regressions.py:5 | ControlFlowNode for ClassExpr | import | class Queue | builtin-class type | +| r_regressions.py:5 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| r_regressions.py:7 | ControlFlowNode for FunctionExpr | import | Function Queue.__init__ | builtin-class function | +| r_regressions.py:9 | ControlFlowNode for Attribute | runtime | Method(Function Queue._after_fork, self instance of Queue) | builtin-class method | +| r_regressions.py:9 | ControlFlowNode for Attribute() | runtime | None | builtin-class NoneType | +| r_regressions.py:9 | ControlFlowNode for self | runtime | self instance of Queue | class Queue | +| r_regressions.py:11 | ControlFlowNode for FunctionExpr | import | Function Queue._after_fork | builtin-class function | +| r_regressions.py:12 | ControlFlowNode for False | code/r_regressions.py:9 from runtime | bool False | builtin-class bool | +| r_regressions.py:12 | ControlFlowNode for self | code/r_regressions.py:9 from runtime | self instance of Queue | class Queue | +| r_regressions.py:13 | ControlFlowNode for None | code/r_regressions.py:9 from runtime | None | builtin-class NoneType | +| r_regressions.py:13 | ControlFlowNode for self | code/r_regressions.py:9 from runtime | self instance of Queue | class Queue | +| r_regressions.py:15 | ControlFlowNode for FunctionExpr | import | Function Queue.close | builtin-class function | +| r_regressions.py:16 | ControlFlowNode for True | runtime | bool True | builtin-class bool | +| r_regressions.py:16 | ControlFlowNode for self | runtime | self instance of Queue | class Queue | +| r_regressions.py:18 | ControlFlowNode for self | runtime | self instance of Queue | class Queue | +| r_regressions.py:20 | ControlFlowNode for self | runtime | self instance of Queue | class Queue | +| r_regressions.py:22 | ControlFlowNode for None | runtime | None | builtin-class NoneType | +| r_regressions.py:22 | ControlFlowNode for self | runtime | self instance of Queue | class Queue | +| r_regressions.py:27 | ControlFlowNode for FunctionExpr | import | Function f | builtin-class function | +| r_regressions.py:27 | ControlFlowNode for IntegerLiteral | import | int 0 | builtin-class int | +| r_regressions.py:27 | ControlFlowNode for None | import | None | builtin-class NoneType | +| r_regressions.py:31 | ControlFlowNode for y | runtime | None | builtin-class NoneType | +| r_regressions.py:33 | ControlFlowNode for y | runtime | None | builtin-class NoneType | +| r_regressions.py:35 | ControlFlowNode for UnaryExpr | runtime | bool False | builtin-class bool | +| r_regressions.py:35 | ControlFlowNode for UnaryExpr | runtime | bool True | builtin-class bool | +| r_regressions.py:36 | ControlFlowNode for z | runtime | int 0 | builtin-class int | +| r_regressions.py:42 | ControlFlowNode for FunctionExpr | import | Function find_library | builtin-class function | +| r_regressions.py:43 | ControlFlowNode for List | runtime | List | builtin-class list | +| r_regressions.py:46 | ControlFlowNode for FunctionExpr | import | Function fail | builtin-class function | +| r_regressions.py:49 | ControlFlowNode for ClassExpr | import | class C | builtin-class type | +| r_regressions.py:49 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| r_regressions.py:51 | ControlFlowNode for FunctionExpr | import | Function C.fail | builtin-class function | +| r_regressions.py:52 | ControlFlowNode for fail | runtime | Function fail | builtin-class function | +| r_regressions.py:52 | ControlFlowNode for fail() | runtime | None | builtin-class NoneType | +| r_regressions.py:58 | ControlFlowNode for FunctionExpr | import | Function method_decorator | builtin-class function | +| r_regressions.py:58 | ControlFlowNode for Str | import | '' | builtin-class str | +| r_regressions.py:61 | ControlFlowNode for FunctionExpr | code/r_regressions.py:85 from import | Function method_decorator._dec | builtin-class function | +| r_regressions.py:61 | ControlFlowNode for FunctionExpr | runtime | Function method_decorator._dec | builtin-class function | +| r_regressions.py:62 | ControlFlowNode for isinstance | code/r_regressions.py:85 from import | Builtin-function isinstance | builtin-class builtin_function_or_method | +| r_regressions.py:62 | ControlFlowNode for isinstance() | code/r_regressions.py:85 from import | bool True | builtin-class bool | +| r_regressions.py:62 | ControlFlowNode for obj | code/r_regressions.py:85 from import | class TestFirst | builtin-class type | +| r_regressions.py:62 | ControlFlowNode for type | code/r_regressions.py:85 from import | builtin-class type | builtin-class type | +| r_regressions.py:63 | ControlFlowNode for is_class | code/r_regressions.py:85 from import | bool True | builtin-class bool | +| r_regressions.py:68 | ControlFlowNode for FunctionExpr | code/r_regressions.py:85 from import | Function method_decorator._dec._wrapper | builtin-class function | +| r_regressions.py:72 | ControlFlowNode for is_class | code/r_regressions.py:85 from import | bool True | builtin-class bool | +| r_regressions.py:73 | ControlFlowNode for _wrapper | code/r_regressions.py:85 from import | Function method_decorator._dec._wrapper | builtin-class function | +| r_regressions.py:73 | ControlFlowNode for obj | code/r_regressions.py:85 from import | class TestFirst | builtin-class type | +| r_regressions.py:73 | ControlFlowNode for setattr | code/r_regressions.py:85 from import | Builtin-function setattr | builtin-class builtin_function_or_method | +| r_regressions.py:73 | ControlFlowNode for setattr() | code/r_regressions.py:85 from import | None | builtin-class NoneType | +| r_regressions.py:74 | ControlFlowNode for obj | code/r_regressions.py:85 from import | class TestFirst | builtin-class type | +| r_regressions.py:78 | ControlFlowNode for _dec | code/r_regressions.py:85 from import | Function method_decorator._dec | builtin-class function | +| r_regressions.py:78 | ControlFlowNode for _dec | runtime | Function method_decorator._dec | builtin-class function | +| r_regressions.py:80 | ControlFlowNode for FunctionExpr | import | Function deco | builtin-class function | +| r_regressions.py:81 | ControlFlowNode for FunctionExpr | runtime | Function deco._wrapper | builtin-class function | +| r_regressions.py:83 | ControlFlowNode for _wrapper | runtime | Function deco._wrapper | builtin-class function | +| r_regressions.py:85 | ControlFlowNode for Str | import | 'method' | builtin-class str | +| r_regressions.py:85 | ControlFlowNode for deco | import | Function deco | builtin-class function | +| r_regressions.py:85 | ControlFlowNode for method_decorator | import | Function method_decorator | builtin-class function | +| r_regressions.py:85 | ControlFlowNode for method_decorator() | import | Function method_decorator._dec | builtin-class function | +| r_regressions.py:85 | ControlFlowNode for method_decorator()() | import | class TestFirst | builtin-class type | +| r_regressions.py:86 | ControlFlowNode for ClassExpr | import | class TestFirst | builtin-class type | +| r_regressions.py:86 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| r_regressions.py:87 | ControlFlowNode for FunctionExpr | import | Function TestFirst.method | builtin-class function | +| r_regressions.py:88 | ControlFlowNode for Str | code/r_regressions.py:90 from import | 'hello world' | builtin-class str | +| r_regressions.py:88 | ControlFlowNode for Str | runtime | 'hello world' | builtin-class str | +| r_regressions.py:90 | ControlFlowNode for Attribute | import | Method(Function TestFirst.method, TestFirst()) | builtin-class method | +| r_regressions.py:90 | ControlFlowNode for Attribute() | import | 'hello world' | builtin-class str | +| r_regressions.py:90 | ControlFlowNode for TestFirst | import | class TestFirst | builtin-class type | +| r_regressions.py:90 | ControlFlowNode for TestFirst() | import | TestFirst() | class TestFirst | +| r_regressions.py:93 | ControlFlowNode for ImportExpr | import | Module sys | builtin-class module | +| r_regressions.py:95 | ControlFlowNode for Attribute | import | ('_ast', '_bisect', '_codecs', '_collections', '_datetime', '_elementtree', '_functools', '_heapq', '_imp', '_io', '_locale', '_md5', '_operator', '_pickle', '_posixsubprocess', '_random', '_sha1', '_sha256', '_sha512', '_signal', '_socket', '_sre', '_stat', '_string', '_struct', '_symtable', '_thread', '_tracemalloc', '_warnings', '_weakref', 'array', 'atexit', 'binascii', 'builtins', 'errno', 'faulthandler', 'fcntl', 'gc', 'grp', 'itertools', 'marshal', 'math', 'posix', 'pwd', 'pyexpat', 'select', 'spwd', 'sys', 'syslog', 'time', 'unicodedata', 'xxsubtype', 'zipimport', 'zlib') | builtin-class tuple | +| r_regressions.py:95 | ControlFlowNode for sys | import | Module sys | builtin-class module | +| r_regressions.py:97 | ControlFlowNode for Compare | import | bool False | builtin-class bool | +| r_regressions.py:97 | ControlFlowNode for Compare | import | bool True | builtin-class bool | +| r_regressions.py:97 | ControlFlowNode for Str | import | 'time' | builtin-class str | +| r_regressions.py:97 | ControlFlowNode for _names | import | ('_ast', '_bisect', '_codecs', '_collections', '_datetime', '_elementtree', '_functools', '_heapq', '_imp', '_io', '_locale', '_md5', '_operator', '_pickle', '_posixsubprocess', '_random', '_sha1', '_sha256', '_sha512', '_signal', '_socket', '_sre', '_stat', '_string', '_struct', '_symtable', '_thread', '_tracemalloc', '_warnings', '_weakref', 'array', 'atexit', 'binascii', 'builtins', 'errno', 'faulthandler', 'fcntl', 'gc', 'grp', 'itertools', 'marshal', 'math', 'posix', 'pwd', 'pyexpat', 'select', 'spwd', 'sys', 'syslog', 'time', 'unicodedata', 'xxsubtype', 'zipimport', 'zlib') | builtin-class tuple | +| r_regressions.py:98 | ControlFlowNode for ImportExpr | import | Module time | builtin-class module | +| r_regressions.py:100 | ControlFlowNode for C | import | class C | builtin-class type | +| r_regressions.py:100 | ControlFlowNode for C() | import | C() | class C | +| r_regressions.py:104 | ControlFlowNode for gv | import | C() | class C | +| r_regressions.py:106 | ControlFlowNode for FunctionExpr | import | Function mod_gv | builtin-class function | +| r_regressions.py:107 | ControlFlowNode for gv | runtime | C() | class C | +| s_scopes.py:4 | ControlFlowNode for True | import | bool True | builtin-class bool | +| s_scopes.py:7 | ControlFlowNode for ClassExpr | import | class C2 | builtin-class type | +| s_scopes.py:7 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| s_scopes.py:9 | ControlFlowNode for int | import | builtin-class int | builtin-class type | +| s_scopes.py:10 | ControlFlowNode for float | import | bool True | builtin-class bool | +| s_scopes.py:10 | ControlFlowNode for float | import | builtin-class float | builtin-class type | +| s_scopes.py:12 | ControlFlowNode for IntegerLiteral | import | int 0 | builtin-class int | +| s_scopes.py:15 | ControlFlowNode for FloatLiteral | import | float 1.0 | builtin-class float | +| s_scopes.py:17 | ControlFlowNode for None | import | None | builtin-class NoneType | +| s_scopes.py:18 | ControlFlowNode for int | import | int 0 | builtin-class int | +| s_scopes.py:19 | ControlFlowNode for str | import | builtin-class str | builtin-class type | +| s_scopes.py:19 | ControlFlowNode for str | import | float 1.0 | builtin-class float | +| s_scopes.py:20 | ControlFlowNode for float | import | None | builtin-class NoneType | +| s_scopes.py:20 | ControlFlowNode for float | import | bool True | builtin-class bool | +| s_scopes.py:20 | ControlFlowNode for float | import | builtin-class float | builtin-class type | +| s_scopes.py:23 | ControlFlowNode for int | import | builtin-class int | builtin-class type | +| s_scopes.py:24 | ControlFlowNode for float | import | bool True | builtin-class bool | +| s_scopes.py:24 | ControlFlowNode for float | import | builtin-class float | builtin-class type | +| t_type.py:1 | ControlFlowNode for ImportExpr | import | Module sys | builtin-class module | +| t_type.py:3 | ControlFlowNode for ClassExpr | import | class C | builtin-class type | +| t_type.py:3 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| t_type.py:6 | ControlFlowNode for C | import | class C | builtin-class type | +| t_type.py:6 | ControlFlowNode for C() | import | C() | class C | +| t_type.py:6 | ControlFlowNode for type | import | builtin-class type | builtin-class type | +| t_type.py:6 | ControlFlowNode for type() | import | class C | builtin-class type | +| t_type.py:7 | ControlFlowNode for sys | import | Module sys | builtin-class module | +| t_type.py:7 | ControlFlowNode for type | import | builtin-class type | builtin-class type | +| t_type.py:7 | ControlFlowNode for type() | import | builtin-class module | builtin-class type | +| t_type.py:9 | ControlFlowNode for type | import | builtin-class type | builtin-class type | +| t_type.py:10 | ControlFlowNode for Dict | import | Dict | builtin-class dict | +| t_type.py:10 | ControlFlowNode for Tuple | import | (builtin-class object) | builtin-class tuple | +| t_type.py:10 | ControlFlowNode for object | import | builtin-class object | builtin-class type | +| t_type.py:10 | ControlFlowNode for type | import | builtin-class type | builtin-class type | +| t_type.py:10 | ControlFlowNode for type() | import | type() | builtin-class type | +| t_type.py:12 | ControlFlowNode for FunctionExpr | import | Function k | builtin-class function | +| t_type.py:13 | ControlFlowNode for C | runtime | class C | builtin-class type | +| t_type.py:13 | ControlFlowNode for C() | runtime | C() | class C | +| t_type.py:13 | ControlFlowNode for type | runtime | builtin-class type | builtin-class type | +| t_type.py:13 | ControlFlowNode for type() | runtime | class C | builtin-class type | +| t_type.py:14 | ControlFlowNode for sys | runtime | Module sys | builtin-class module | +| t_type.py:14 | ControlFlowNode for type | runtime | builtin-class type | builtin-class type | +| t_type.py:14 | ControlFlowNode for type() | runtime | builtin-class module | builtin-class type | +| t_type.py:15 | ControlFlowNode for type | runtime | builtin-class type | builtin-class type | +| t_type.py:16 | ControlFlowNode for Dict | runtime | Dict | builtin-class dict | +| t_type.py:16 | ControlFlowNode for Tuple | runtime | (builtin-class object) | builtin-class tuple | +| t_type.py:16 | ControlFlowNode for object | runtime | builtin-class object | builtin-class type | +| t_type.py:16 | ControlFlowNode for type | runtime | builtin-class type | builtin-class type | +| t_type.py:16 | ControlFlowNode for type() | runtime | type() | builtin-class type | +| u_paired_values.py:2 | ControlFlowNode for FunctionExpr | import | Function return_if_true | builtin-class function | +| u_paired_values.py:3 | ControlFlowNode for cond | code/u_paired_values.py:8 from code/u_paired_values.py:11 from import | bool True | builtin-class bool | +| u_paired_values.py:3 | ControlFlowNode for cond | code/u_paired_values.py:8 from code/u_paired_values.py:14 from import | bool False | builtin-class bool | +| u_paired_values.py:3 | ControlFlowNode for cond | code/u_paired_values.py:8 from runtime | bool False | builtin-class bool | +| u_paired_values.py:3 | ControlFlowNode for cond | code/u_paired_values.py:8 from runtime | bool True | builtin-class bool | +| u_paired_values.py:4 | ControlFlowNode for val | code/u_paired_values.py:8 from code/u_paired_values.py:11 from import | int 1 | builtin-class int | +| u_paired_values.py:4 | ControlFlowNode for val | code/u_paired_values.py:8 from runtime | int 1 | builtin-class int | +| u_paired_values.py:5 | ControlFlowNode for Exception | code/u_paired_values.py:8 from code/u_paired_values.py:14 from import | builtin-class Exception | builtin-class type | +| u_paired_values.py:5 | ControlFlowNode for Exception | code/u_paired_values.py:8 from runtime | builtin-class Exception | builtin-class type | +| u_paired_values.py:5 | ControlFlowNode for Exception | runtime | builtin-class Exception | builtin-class type | +| u_paired_values.py:5 | ControlFlowNode for Exception() | code/u_paired_values.py:8 from code/u_paired_values.py:14 from import | Exception() | builtin-class Exception | +| u_paired_values.py:5 | ControlFlowNode for Exception() | code/u_paired_values.py:8 from runtime | Exception() | builtin-class Exception | +| u_paired_values.py:5 | ControlFlowNode for Exception() | runtime | Exception() | builtin-class Exception | +| u_paired_values.py:7 | ControlFlowNode for FunctionExpr | import | Function test | builtin-class function | +| u_paired_values.py:8 | ControlFlowNode for False | code/u_paired_values.py:14 from import | bool False | builtin-class bool | +| u_paired_values.py:8 | ControlFlowNode for False | runtime | bool False | builtin-class bool | +| u_paired_values.py:8 | ControlFlowNode for IfExp | code/u_paired_values.py:11 from import | int 1 | builtin-class int | +| u_paired_values.py:8 | ControlFlowNode for IfExp | runtime | int 1 | builtin-class int | +| u_paired_values.py:8 | ControlFlowNode for IntegerLiteral | code/u_paired_values.py:11 from import | int 1 | builtin-class int | +| u_paired_values.py:8 | ControlFlowNode for IntegerLiteral | code/u_paired_values.py:14 from import | int 2 | builtin-class int | +| u_paired_values.py:8 | ControlFlowNode for IntegerLiteral | runtime | int 1 | builtin-class int | +| u_paired_values.py:8 | ControlFlowNode for IntegerLiteral | runtime | int 2 | builtin-class int | +| u_paired_values.py:8 | ControlFlowNode for True | code/u_paired_values.py:11 from import | bool True | builtin-class bool | +| u_paired_values.py:8 | ControlFlowNode for True | runtime | bool True | builtin-class bool | +| u_paired_values.py:8 | ControlFlowNode for cond | code/u_paired_values.py:11 from import | bool True | builtin-class bool | +| u_paired_values.py:8 | ControlFlowNode for cond | code/u_paired_values.py:14 from import | bool False | builtin-class bool | +| u_paired_values.py:8 | ControlFlowNode for return_if_true | code/u_paired_values.py:11 from import | Function return_if_true | builtin-class function | +| u_paired_values.py:8 | ControlFlowNode for return_if_true | code/u_paired_values.py:14 from import | Function return_if_true | builtin-class function | +| u_paired_values.py:8 | ControlFlowNode for return_if_true | runtime | Function return_if_true | builtin-class function | +| u_paired_values.py:8 | ControlFlowNode for return_if_true() | code/u_paired_values.py:11 from import | int 1 | builtin-class int | +| u_paired_values.py:8 | ControlFlowNode for return_if_true() | runtime | int 1 | builtin-class int | +| u_paired_values.py:9 | ControlFlowNode for x | code/u_paired_values.py:11 from import | int 1 | builtin-class int | +| u_paired_values.py:9 | ControlFlowNode for x | runtime | int 1 | builtin-class int | +| u_paired_values.py:11 | ControlFlowNode for True | import | bool True | builtin-class bool | +| u_paired_values.py:11 | ControlFlowNode for test | import | Function test | builtin-class function | +| u_paired_values.py:11 | ControlFlowNode for test() | import | int 1 | builtin-class int | +| u_paired_values.py:12 | ControlFlowNode for y | import | int 1 | builtin-class int | +| u_paired_values.py:14 | ControlFlowNode for False | import | bool False | builtin-class bool | +| u_paired_values.py:14 | ControlFlowNode for test | import | Function test | builtin-class function | diff --git a/python/ql/test/library-tests/PointsTo/new/Values.ql b/python/ql/test/library-tests/PointsTo/new/Values.ql index f1f3dd43aa49..64adbab58e0b 100644 --- a/python/ql/test/library-tests/PointsTo/new/Values.ql +++ b/python/ql/test/library-tests/PointsTo/new/Values.ql @@ -6,4 +6,4 @@ import Util from ControlFlowNode f, Context ctx, Value v, ControlFlowNode origin where f.pointsTo(ctx, v, origin) -select locate(f.getLocation(), "c"), f.toString(), ctx, v, v.getClass() +select locate(f.getLocation(), "abeghijklmnpqrstu"), f.toString(), ctx, v, v.getClass() diff --git a/python/ql/test/library-tests/types/classes/FailedInference.expected b/python/ql/test/library-tests/types/classes/FailedInference.expected index 916075a656c2..a8103a8847e1 100644 --- a/python/ql/test/library-tests/types/classes/FailedInference.expected +++ b/python/ql/test/library-tests/types/classes/FailedInference.expected @@ -1,25 +1,35 @@ -| circular_inheritance.py:33:1:33:11 | class A | Missing base 0 | -| circular_inheritance.py:37:1:37:11 | class B | Missing base 0 | -| circular_inheritance.py:40:1:40:72 | class D | Duplicate bases classes | -| circular_inheritance.py:43:1:43:41 | class E | Duplicate bases classes | -| circular_inheritance.py:43:1:43:41 | class E | Failed inference for base class at position 0 | -| circular_inheritance.py:43:1:43:41 | class E | Failed inference for base class at position 1 | -| circular_inheritance.py:43:1:43:41 | class E | Failed inference for base class at position 2 | -| circular_inheritance.py:43:1:43:41 | class E | Failed inference for base class at position 3 | -| circular_inheritance.py:43:1:43:41 | class E | Failed inference for base class at position 4 | -| circular_inheritance.py:43:1:43:41 | class E | Failed inference for base class at position 5 | -| circular_inheritance.py:43:1:43:41 | class E | Failed inference for base class at position 6 | -| circular_inheritance.py:43:1:43:41 | class E | Failed inference for base class at position 7 | -| circular_inheritance.py:43:1:43:41 | class E | Failed inference for base class at position 8 | -| circular_inheritance.py:43:1:43:41 | class E | Failed inference for base class at position 9 | -| circular_inheritance.py:43:1:43:41 | class E | Failed inference for base class at position 10 | -| circular_inheritance.py:47:1:47:19 | class G | Missing base 0 | -| circular_inheritance.py:50:1:50:19 | class J | Missing base 0 | -| circular_inheritance.py:54:1:54:11 | class M | Missing base 0 | -| circular_inheritance.py:57:1:57:11 | class L | Missing base 0 | -| circular_inheritance.py:61:1:61:19 | class S | Missing base 0 | -| circular_inheritance.py:64:1:64:19 | class R | Missing base 0 | -| mutual_inheritance.py:4:1:4:11 | class C | Missing base 0 | -| mutual_inheritance.py:8:1:8:19 | class H | Missing base 0 | -| mutual_inheritance.py:12:1:12:11 | class N | Missing base 0 | -| mutual_inheritance.py:15:1:15:19 | class T | Missing base 0 | +| class A | Missing base 0 | +| class B | Failed inference for base class at position 0 | +| class B | Failed to compute MRO | +| class C | Failed inference for base class at position 0 | +| class C | Failed to compute MRO | +| class D | Duplicate bases classes | +| class E | Duplicate bases classes | +| class E | Failed inference for base class at position 0 | +| class E | Failed inference for base class at position 1 | +| class E | Failed inference for base class at position 2 | +| class E | Failed inference for base class at position 3 | +| class E | Failed inference for base class at position 4 | +| class E | Failed inference for base class at position 5 | +| class E | Failed inference for base class at position 6 | +| class E | Failed inference for base class at position 7 | +| class E | Failed inference for base class at position 8 | +| class E | Failed inference for base class at position 9 | +| class E | Failed inference for base class at position 10 | +| class G | Failed inference for base class at position 0 | +| class G | Failed to compute MRO | +| class H | Failed inference for base class at position 0 | +| class H | Failed to compute MRO | +| class J | Missing base 0 | +| class L | Failed inference for base class at position 0 | +| class L | Failed to compute MRO | +| class M | Failed inference for base class at position 0 | +| class M | Failed to compute MRO | +| class N | Failed inference for base class at position 0 | +| class N | Failed to compute MRO | +| class R | Failed inference for base class at position 0 | +| class R | Failed to compute MRO | +| class S | Failed inference for base class at position 0 | +| class S | Failed to compute MRO | +| class T | Failed inference for base class at position 0 | +| class T | Failed to compute MRO | diff --git a/python/ql/test/library-tests/types/classes/FailedInference.ql b/python/ql/test/library-tests/types/classes/FailedInference.ql index 129c17ffd9df..df5e2ccf14fd 100644 --- a/python/ql/test/library-tests/types/classes/FailedInference.ql +++ b/python/ql/test/library-tests/types/classes/FailedInference.ql @@ -2,10 +2,10 @@ import python import semmle.python.pointsto.PointsTo -from ClassObject cls, string reason +from ClassValue cls, string reason where -PointsTo::Types::failed_inference(cls, reason) +Types::failedInference(cls, reason) select cls, reason From 96eaf815da8078f504b9969a435a06c746e21172 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 17 Apr 2019 15:26:20 +0100 Subject: [PATCH 062/128] Python: Fix up state-tracking library. --- .../ql/src/semmle/python/dataflow/StateTracking.qll | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/ql/src/semmle/python/dataflow/StateTracking.qll b/python/ql/src/semmle/python/dataflow/StateTracking.qll index 39f240f6388b..760aaf25dc3a 100644 --- a/python/ql/src/semmle/python/dataflow/StateTracking.qll +++ b/python/ql/src/semmle/python/dataflow/StateTracking.qll @@ -11,6 +11,7 @@ import python private import semmle.python.pointsto.Base private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext +private import semmle.python.objects.ObjectInternal /** A state that should be tracked. */ abstract class TrackableState extends string { @@ -106,23 +107,23 @@ module StateTracking { exists(int n | f = b.getNode(n) and appliesToNode(state, b.getNode(n-1), ctx, sense) and - not exists(PyFunctionObject func, Context callee | + not exists(PythonFunctionObjectInternal func, Context callee | callee.fromCall(f, func, ctx) ) ) ) or /* Function entry via call */ - exists(FunctionObject func, CallNode call, Context caller | + exists(PythonFunctionObjectInternal func, CallNode call, Context caller | ctx.fromCall(call, func, caller) and - func.getFunction().getEntryNode() = f and + func.getScope().getEntryNode() = f and appliesToNode(state, call.getAPredecessor(), caller, sense) ) or /* Function return */ - exists(PyFunctionObject func, Context callee | + exists(PythonFunctionObjectInternal func, Context callee | callee.fromCall(f, func, ctx) and - appliesToNode(state, func.getFunction().getANormalExit(), callee, sense) + appliesToNode(state, func.getScope().getANormalExit(), callee, sense) ) or /* Other scope entries */ From 8a2fb54c49d87295ef8199498a6e1633eb228f43 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 17 Apr 2019 15:26:52 +0100 Subject: [PATCH 063/128] Python points-to. Track bitwise or-ing of small integer flags. --- python/ql/src/semmle/python/Flow.qll | 10 ++++++++ .../src/semmle/python/pointsto/PointsTo.qll | 25 +++++++++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll index 1b70a3b16639..da4cd3c3f36e 100755 --- a/python/ql/src/semmle/python/Flow.qll +++ b/python/ql/src/semmle/python/Flow.qll @@ -658,6 +658,16 @@ class BinaryExprNode extends ControlFlowNode { result = this.getNode().getOp() } + /** Whether left and right are a pair of operands for this binary expression */ + predicate operands(ControlFlowNode left, Operator op, ControlFlowNode right) { + exists(BinaryExpr b, Expr eleft, Expr eright | + this.getNode() = b and left.getNode() = eleft and right.getNode() = eright | + eleft = b.getLeft() and eright = b.getRight() and op = b.getOp() + ) and + left.getBasicBlock().dominates(this.getBasicBlock()) and + right.getBasicBlock().dominates(this.getBasicBlock()) + } + } /** A control flow node corresponding to a boolean shortcut (and/or) operation */ diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 0f0427cab326..1a3e6c61572f 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1119,11 +1119,26 @@ module Expressions { */ pragma [noinline] predicate binaryPointsTo(BinaryExprNode b, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode operand, ObjectInternal opvalue) { - // TO DO... - // Track some integer values through `|` and the types of some objects - operand = b.getAnOperand() and - PointsToInternal::pointsTo(operand, context, opvalue, _) and - value = ObjectInternal::unknown() and origin = b + origin = b and + exists(ControlFlowNode left, Operator op, ControlFlowNode right | + b.operands(left, op, right) + | + not op instanceof BitOr and + (operand = left or operand = right) and + PointsToInternal::pointsTo(operand, context, opvalue, _) and + value = ObjectInternal::unknown() + or + op instanceof BitOr and + exists(ObjectInternal lobj, ObjectInternal robj | + PointsToInternal::pointsTo(left, context, lobj, _) and + PointsToInternal::pointsTo(right, context, robj, _) and + value = TInt(lobj.intValue().bitOr(robj.intValue())) + | + left = operand and opvalue = lobj + or + right = operand and opvalue = robj + ) + ) } pragma [noinline] From f51a2d9ec45539425aa5da4205fe5a5cc6a93ac2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 17 Apr 2019 18:04:19 +0100 Subject: [PATCH 064/128] Python points-to: Fix up test-evaluate for ABCs and tests involving type(). --- .../src/semmle/python/objects/ObjectAPI.qll | 3 + .../ql/src/semmle/python/objects/TObject.qll | 10 ++ .../src/semmle/python/pointsto/PointsTo.qll | 93 +++++++++---------- .../PointsTo/subclass/Checks.expected | 13 +++ .../library-tests/PointsTo/subclass/Checks.ql | 8 ++ .../PointsTo/subclass/TestEvaluate.expected | 11 +++ .../PointsTo/subclass/TestEvaluate.ql | 12 +++ .../PointsTo/subclass/Values.expected | 66 +++++++++++++ .../library-tests/PointsTo/subclass/Values.ql | 10 ++ .../library-tests/PointsTo/subclass/test.py | 18 ++++ .../PointsTo/super/SuperMethodCall.ql | 9 +- .../ql/test/library-tests/classes/mro/C3.ql | 4 +- 12 files changed, 204 insertions(+), 53 deletions(-) create mode 100644 python/ql/test/library-tests/PointsTo/subclass/Checks.expected create mode 100644 python/ql/test/library-tests/PointsTo/subclass/Checks.ql create mode 100644 python/ql/test/library-tests/PointsTo/subclass/TestEvaluate.expected create mode 100644 python/ql/test/library-tests/PointsTo/subclass/TestEvaluate.ql create mode 100644 python/ql/test/library-tests/PointsTo/subclass/Values.expected create mode 100644 python/ql/test/library-tests/PointsTo/subclass/Values.ql create mode 100644 python/ql/test/library-tests/PointsTo/subclass/test.py diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 836f9ec65140..916f32535f41 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -53,6 +53,9 @@ class Value extends TObject { result = this.(ObjectInternal).getSource() } + predicate isBuiltin() { + this.(ObjectInternal).isBuiltin() + } } class ModuleValue extends Value { diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index c49896e2fabc..a1c6402d409f 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -365,5 +365,15 @@ library class ClassDecl extends @py_object { this instanceof Builtin } + predicate isAbstractBaseClass(string name) { + exists(Module m | + m.getName() = "_abcoll" + or + m.getName() = "_collections_abc" + | + this.getClass().getScope() = m and + this.getName() = name + ) + } } diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 1a3e6c61572f..0335986b0a93 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -817,24 +817,7 @@ module InterProceduralPointsTo { PointsToInternal::pointsTo(f.getArg(0), context, value, origin) ) or - value = call_to_type(f, context) and - ( - value.isBuiltin() and origin = f - or - origin = value.getOrigin() - or - value = ObjectInternal::unknownClass() and origin = f - ) - } - - pragma [noinline] - private ObjectInternal call_to_type(CallNode f, PointsToContext context) { - count(f.getArg(_)) = 1 and - call(f, context, ObjectInternal::builtin("type")) and - exists(ObjectInternal arg | - PointsToInternal::pointsTo(f.getArg(0), context, arg, _) and - result = arg.getClass() - ) + Expressions::typeCallPointsTo(f, context, value, origin, _, _) } /** Points-to for parameter. `def foo(param): ...`. */ @@ -1174,6 +1157,16 @@ module Expressions { origin = call } + pragma [noinline] + predicate typeCallPointsTo(CallNode call, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode arg, ObjectInternal argvalue) { + not exists(call.getArg(1)) and + arg = call.getArg(0) and + InterProceduralPointsTo::call(call, context, ObjectInternal::builtin("type")) and + PointsToInternal::pointsTo(arg, context, argvalue, _) and + value = argvalue.getClass() and + origin = CfgOrigin::fromObject(value).asCfgNodeOrHere(call) + } + pragma [noinline] private predicate lenCallPointsTo(CallNode call, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode arg, ObjectInternal argvalue) { len_call(call, arg, context, argvalue) and @@ -1317,6 +1310,8 @@ module Expressions { or lenCallPointsTo(expr, context, value, origin, subexpr, subvalue) or + typeCallPointsTo(expr, context, value, origin, subexpr, subvalue) + or getattrPointsTo(expr, context, value, origin, subexpr, subvalue) or value = ObjectInternal::bool(evaluatesTo(expr, context, subexpr, subvalue)) and origin = expr @@ -1754,7 +1749,9 @@ cached module Types { cached boolean improperSubclass(ObjectInternal sub, ObjectInternal sup) { sub = sup and result = true or - result = mroContains(Types::getMro(sub), sup, 0) + result = true and mroContains(Types::getMro(sub), sup) + or + result = false and mroDoesnotContain(Types::getMro(sub), sup, 0) or result = tupleSubclass(sub, sup, 0) } @@ -1768,32 +1765,47 @@ cached module Types { ) } - private boolean mroContains(ClassList mro, ClassObjectInternal sup, int n) { + private predicate mroContains(ClassList mro, ClassObjectInternal sup) { + mro.contains(sup) + or + exists(ClassDecl item, ClassDecl sdecl | + item = mro.getAnItem().getClassDeclaration() and + sdecl = sup.getClassDeclaration() and + is_abstract_subclass(item, sdecl) + ) + } + + private predicate mroDoesnotContain(ClassList mro, ClassObjectInternal sup, int n) { exists(ClassObjectInternal cls | Expressions::requireSubClass(cls, sup) and mro = getMro(cls) ) and ( - n = mro.length() and result = false - or - mro.getItem(n) = sup and result = true - or - mro.getItem(n) = abc_to_concrete(sup) and result = true - or - mro.getItem(n) != sup and sup != AbstractBaseClass::named("Iterable") and - mro.getItem(n) != abc_to_concrete(sup) and result = mroContains(mro, sup, n+1) - or - sup = AbstractBaseClass::named("Iterable") and result = mro.getItem(n).isIterableSubclass() + n = mro.length() + or + mroDoesnotContain(mro, sup, n+1) and + mro.getItem(n) != sup and + exists(ClassDecl item, ClassDecl sdecl | + item = mro.getItem(n).getClassDeclaration() and + sdecl = sup.getClassDeclaration() and + not is_abstract_subclass(item, sdecl) + ) ) } - private ClassObjectInternal abc_to_concrete(ClassObjectInternal c) { - c = AbstractBaseClass::named("Sequence") and result = ObjectInternal::builtin("list") + private predicate is_abstract_subclass(ClassDecl cls, ClassDecl sup) { + cls = Builtin::builtin("list") and sup.isAbstractBaseClass("Sequence") + or + cls = Builtin::builtin("set") and sup.isAbstractBaseClass("Set") or - c = AbstractBaseClass::named("Set") and result = ObjectInternal::builtin("set") + cls = Builtin::builtin("dict") and sup.isAbstractBaseClass("Mapping") or - c = AbstractBaseClass::named("Mapping") and result = ObjectInternal::builtin("dict") + cls = Builtin::builtin("list") and sup.isAbstractBaseClass("Iterable") + or + cls = Builtin::builtin("set") and sup.isAbstractBaseClass("Iterable") + or + cls = Builtin::builtin("dict") and sup.isAbstractBaseClass("Iterable") } cached boolean hasAttr(ObjectInternal cls, string name) { @@ -1822,19 +1834,6 @@ cached module Types { } -module AbstractBaseClass { - - ClassObjectInternal named(string name) { - exists(ModuleObjectInternal m | - m.getName() = "_abcoll" - or - m.getName() = "_collections_abc" - | - m.attribute(name, result, _) - ) - } -} - module AttributePointsTo { predicate pointsTo(AttrNode f, Context context, ObjectInternal value, ControlFlowNode origin) { diff --git a/python/ql/test/library-tests/PointsTo/subclass/Checks.expected b/python/ql/test/library-tests/PointsTo/subclass/Checks.expected new file mode 100644 index 000000000000..51fe4db79ad7 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/subclass/Checks.expected @@ -0,0 +1,13 @@ +| builtin-class dict | builtin-class dict | +| builtin-class dict | builtin-class list | +| builtin-class int | (builtin-class float, builtin-class dict) | +| builtin-class int | (builtin-class list, builtin-class int) | +| builtin-class int | (builtin-class list, builtin-class int) | +| builtin-class int | builtin-class dict | +| builtin-class int | builtin-class float | +| builtin-class int | builtin-class int | +| builtin-class int | builtin-class list | +| builtin-class int | builtin-class object | +| builtin-class int | builtin-class tuple | +| builtin-class tuple | builtin-class int | +| builtin-class tuple | builtin-class tuple | diff --git a/python/ql/test/library-tests/PointsTo/subclass/Checks.ql b/python/ql/test/library-tests/PointsTo/subclass/Checks.ql new file mode 100644 index 000000000000..68921ec97399 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/subclass/Checks.ql @@ -0,0 +1,8 @@ + +import python +import semmle.python.pointsto.PointsTo +import semmle.python.pointsto.PointsTo + +from Value sup, Value cls +where Expressions::requireSubClass(cls, sup) +select cls, sup diff --git a/python/ql/test/library-tests/PointsTo/subclass/TestEvaluate.expected b/python/ql/test/library-tests/PointsTo/subclass/TestEvaluate.expected new file mode 100644 index 000000000000..dc038d1ce790 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/subclass/TestEvaluate.expected @@ -0,0 +1,11 @@ +| 3 | isinstance() | true | x | int 7 | +| 4 | issubclass() | true | x | int 7 | +| 6 | issubclass() | true | d | builtin-class dict | +| 7 | UnaryExpr | true | d | builtin-class dict | +| 10 | isinstance() | false | x | int 0 | +| 10 | isinstance() | true | x | () | +| 13 | isinstance() | false | x | () | +| 13 | isinstance() | true | x | int 0 | +| 14 | isinstance() | true | x | int 0 | +| 15 | issubclass() | true | x | int 0 | +| 16 | issubclass() | false | x | int 0 | diff --git a/python/ql/test/library-tests/PointsTo/subclass/TestEvaluate.ql b/python/ql/test/library-tests/PointsTo/subclass/TestEvaluate.ql new file mode 100644 index 000000000000..fd32bfcbe643 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/subclass/TestEvaluate.ql @@ -0,0 +1,12 @@ + +import python +import semmle.python.pointsto.PointsTo +import semmle.python.objects.ObjectInternal +import semmle.python.pointsto.PointsToContext + + +from ControlFlowNode test, ControlFlowNode use, ObjectInternal val, boolean eval, PointsToContext ctx +where +PointsTo::pointsTo(use, ctx, val, _) and +eval = Conditionals::testEvaluates(test, use, ctx, val, _) +select test.getLocation().getStartLine(), test.getNode().toString(), eval.toString(), use.getNode().toString(), val.toString() diff --git a/python/ql/test/library-tests/PointsTo/subclass/Values.expected b/python/ql/test/library-tests/PointsTo/subclass/Values.expected new file mode 100644 index 000000000000..48ec378ee024 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/subclass/Values.expected @@ -0,0 +1,66 @@ +| 2 | ControlFlowNode for IntegerLiteral | int 7 | 2 | +| 3 | ControlFlowNode for int | builtin-class int | 3 | +| 3 | ControlFlowNode for isinstance | Builtin-function isinstance | 3 | +| 3 | ControlFlowNode for isinstance() | bool True | 3 | +| 3 | ControlFlowNode for x | int 7 | 2 | +| 4 | ControlFlowNode for issubclass | Builtin-function issubclass | 4 | +| 4 | ControlFlowNode for issubclass() | bool True | 4 | +| 4 | ControlFlowNode for object | builtin-class object | 4 | +| 4 | ControlFlowNode for type | builtin-class type | 4 | +| 4 | ControlFlowNode for type() | builtin-class int | 4 | +| 4 | ControlFlowNode for x | int 7 | 2 | +| 5 | ControlFlowNode for dict | builtin-class dict | 5 | +| 6 | ControlFlowNode for d | builtin-class dict | 5 | +| 6 | ControlFlowNode for dict | builtin-class dict | 6 | +| 6 | ControlFlowNode for issubclass | Builtin-function issubclass | 6 | +| 6 | ControlFlowNode for issubclass() | bool True | 6 | +| 7 | ControlFlowNode for UnaryExpr | bool True | 7 | +| 7 | ControlFlowNode for d | builtin-class dict | 5 | +| 7 | ControlFlowNode for issubclass | Builtin-function issubclass | 7 | +| 7 | ControlFlowNode for issubclass() | bool False | 7 | +| 7 | ControlFlowNode for list | builtin-class list | 7 | +| 9 | ControlFlowNode for IfExp | () | 9 | +| 9 | ControlFlowNode for IfExp | int 0 | 9 | +| 9 | ControlFlowNode for IntegerLiteral | int 0 | 9 | +| 9 | ControlFlowNode for Tuple | () | 9 | +| 9 | ControlFlowNode for condition | Unknown value | 9 | +| 10 | ControlFlowNode for isinstance | Builtin-function isinstance | 10 | +| 10 | ControlFlowNode for isinstance() | bool False | 10 | +| 10 | ControlFlowNode for isinstance() | bool True | 10 | +| 10 | ControlFlowNode for tuple | builtin-class tuple | 10 | +| 10 | ControlFlowNode for x | () | 9 | +| 10 | ControlFlowNode for x | int 0 | 9 | +| 12 | ControlFlowNode for IntegerLiteral | int 3 | 12 | +| 12 | ControlFlowNode for isinstance | Builtin-function isinstance | 12 | +| 12 | ControlFlowNode for isinstance() | bool False | 12 | +| 12 | ControlFlowNode for isinstance() | bool True | 12 | +| 12 | ControlFlowNode for unknown | Unknown value | 12 | +| 12 | ControlFlowNode for unknown() | Unknown value | 12 | +| 13 | ControlFlowNode for int | builtin-class int | 13 | +| 13 | ControlFlowNode for isinstance | Builtin-function isinstance | 13 | +| 13 | ControlFlowNode for isinstance() | bool False | 13 | +| 13 | ControlFlowNode for isinstance() | bool True | 13 | +| 13 | ControlFlowNode for x | () | 9 | +| 13 | ControlFlowNode for x | int 0 | 9 | +| 14 | ControlFlowNode for Tuple | (builtin-class list, builtin-class int) | 14 | +| 14 | ControlFlowNode for int | builtin-class int | 14 | +| 14 | ControlFlowNode for isinstance | Builtin-function isinstance | 14 | +| 14 | ControlFlowNode for isinstance() | bool True | 14 | +| 14 | ControlFlowNode for list | builtin-class list | 14 | +| 14 | ControlFlowNode for x | int 0 | 9 | +| 15 | ControlFlowNode for Tuple | (builtin-class list, builtin-class int) | 15 | +| 15 | ControlFlowNode for int | builtin-class int | 15 | +| 15 | ControlFlowNode for issubclass | Builtin-function issubclass | 15 | +| 15 | ControlFlowNode for issubclass() | bool True | 15 | +| 15 | ControlFlowNode for list | builtin-class list | 15 | +| 15 | ControlFlowNode for type | builtin-class type | 15 | +| 15 | ControlFlowNode for type() | builtin-class int | 15 | +| 15 | ControlFlowNode for x | int 0 | 9 | +| 16 | ControlFlowNode for Tuple | (builtin-class float, builtin-class dict) | 16 | +| 16 | ControlFlowNode for dict | builtin-class dict | 16 | +| 16 | ControlFlowNode for float | builtin-class float | 16 | +| 16 | ControlFlowNode for issubclass | Builtin-function issubclass | 16 | +| 16 | ControlFlowNode for issubclass() | bool False | 16 | +| 16 | ControlFlowNode for type | builtin-class type | 16 | +| 16 | ControlFlowNode for type() | builtin-class int | 16 | +| 16 | ControlFlowNode for x | int 0 | 9 | diff --git a/python/ql/test/library-tests/PointsTo/subclass/Values.ql b/python/ql/test/library-tests/PointsTo/subclass/Values.ql new file mode 100644 index 000000000000..3246ef027640 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/subclass/Values.ql @@ -0,0 +1,10 @@ + +import python +import semmle.python.pointsto.PointsTo +import semmle.python.objects.ObjectInternal + +from ControlFlowNode f, ObjectInternal v, ControlFlowNode x + +where PointsTo::pointsTo(f, _, v, x) + +select f.getLocation().getStartLine(), f.toString(), v, x.getLocation().getStartLine() diff --git a/python/ql/test/library-tests/PointsTo/subclass/test.py b/python/ql/test/library-tests/PointsTo/subclass/test.py new file mode 100644 index 000000000000..11ef81f04aa3 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/subclass/test.py @@ -0,0 +1,18 @@ + +x = 7 +assert isinstance(x, int) +assert issubclass(type(x), object) +d = dict +assert issubclass(d, dict) +assert not issubclass(d, list) + +x = 0 if condition else () +if isinstance(x, tuple): + pass +isinstance(3, unknown()) +assert isinstance(x, int) +assert isinstance(x, (list, int)) +assert issubclass(type(x), (list, int)) +if issubclass(type(x), (float, dict)): + pass + diff --git a/python/ql/test/library-tests/PointsTo/super/SuperMethodCall.ql b/python/ql/test/library-tests/PointsTo/super/SuperMethodCall.ql index 4df31ff0478b..f21b102338f5 100644 --- a/python/ql/test/library-tests/PointsTo/super/SuperMethodCall.ql +++ b/python/ql/test/library-tests/PointsTo/super/SuperMethodCall.ql @@ -2,8 +2,9 @@ import python import semmle.python.pointsto.PointsTo import semmle.python.pointsto.PointsToContext +import semmle.python.objects.ObjectInternal -from CallNode call, FunctionObject method -where PointsTo::Test::super_method_call(_, call, _, method) -select call.getLocation().getStartLine(), call.toString(), method.getQualifiedName() - +from CallNode call, SuperInstance sup, BoundMethodObjectInternal bm +where call.getFunction().inferredValue() = bm and +call.getFunction().(AttrNode).getObject().inferredValue() = sup +select call.getLocation().getStartLine(), call.toString(), bm.getFunction().getSource().(FunctionObject).getQualifiedName() \ No newline at end of file diff --git a/python/ql/test/library-tests/classes/mro/C3.ql b/python/ql/test/library-tests/classes/mro/C3.ql index caaa43d3d45a..e433971f1eb3 100644 --- a/python/ql/test/library-tests/classes/mro/C3.ql +++ b/python/ql/test/library-tests/classes/mro/C3.ql @@ -2,8 +2,8 @@ import python import semmle.python.pointsto.MRO -from ClassObject cls +from ClassValue cls where not cls.isBuiltin() -select cls.toString(), new_style_mro(cls) +select cls.toString(), Mro::newStyleMro(cls) From 5ad731a2a8a07695af31a256e025163a84169e68 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 24 Apr 2019 11:36:43 +0100 Subject: [PATCH 065/128] Python points-to. Fix handling of six.add_metaclass. --- .../src/semmle/python/pointsto/PointsTo.qll | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 0335986b0a93..cb6acaf17b79 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1643,19 +1643,17 @@ cached module Types { /** INTERNAL -- Do not use */ cached predicate six_add_metaclass(CallNode decorator_call, ClassObjectInternal decorated, ControlFlowNode metaclass) { - //TO DO... - none() - //exists(CallNode decorator | - // decorator_call.getArg(0) = decorated and - // decorator = decorator_call.getFunction() and - // decorator.getArg(0) = metaclass | - // PointsToInternal::pointsTo(decorator.getFunction(), _, six_add_metaclass_function(), _) - // or - // exists(ModuleObjectInternal six | - // six.getName() = "six" and - // PointsToInternal::pointsTo(decorator.getFunction().(AttrNode).getObject("add_metaclass"), _, six, _) - // ) - //) + exists(CallNode decorator | + PointsToInternal::pointsTo(decorator_call.getArg(0), _, decorated, _) and + decorator = decorator_call.getFunction() and + decorator.getArg(0) = metaclass | + PointsToInternal::pointsTo(decorator.getFunction(), _, six_add_metaclass_function(), _) + or + exists(ModuleObjectInternal six | + six.getName() = "six" and + PointsToInternal::pointsTo(decorator.getFunction().(AttrNode).getObject("add_metaclass"), _, six, _) + ) + ) } private ObjectInternal six_add_metaclass_function() { From 674a3da4b8c4ba9354d9c382b7c1aaf115263350 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 24 Apr 2019 14:19:30 +0100 Subject: [PATCH 066/128] Python points-to: Importing from 'unknown' gives 'unknown'. --- python/ql/src/semmle/python/pointsto/PointsTo.qll | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index cb6acaf17b79..53474661e475 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -607,6 +607,9 @@ module InterModulePointsTo { mod.attribute(name, value, orig) and origin = orig.asCfgNodeOrHere(f) ) + or + PointsToInternal::pointsTo(f.getModule(_), context, ObjectInternal::unknown(), _) and + value = ObjectInternal::unknown() and origin = f } private predicate from_import_imports(ImportMemberNode f, PointsToContext context, ModuleObjectInternal mod, string name) { From 3bb61e74105273e47ea233c68e9cbf4ed1018f05 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 24 Apr 2019 16:50:54 +0100 Subject: [PATCH 067/128] Python points-to: Improve handling of subscripts and sequence inequalities. --- .../src/semmle/python/objects/Callables.qll | 2 + .../ql/src/semmle/python/objects/Classes.qll | 1 + .../src/semmle/python/objects/Constants.qll | 2 + .../src/semmle/python/objects/Descriptors.qll | 6 ++ .../src/semmle/python/objects/Instances.qll | 8 ++ .../ql/src/semmle/python/objects/Modules.qll | 2 + .../semmle/python/objects/ObjectInternal.qll | 17 +++- .../src/semmle/python/objects/Sequences.qll | 78 +++++++++++++++++++ .../ql/src/semmle/python/objects/TObject.qll | 2 + .../src/semmle/python/pointsto/PointsTo.qll | 60 +++++++++++--- .../PointsTo/consts/BooleanConstants.expected | 2 + .../PointsTo/import_time/Pruned.ql | 8 +- 12 files changed, 169 insertions(+), 19 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 59613e895b61..4dec5184181a 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -54,6 +54,8 @@ abstract class CallableObjectInternal extends ObjectInternal { abstract predicate neverReturns(); + override predicate subscriptUnknown() { none() } + } diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 435a99871274..d5786ef30832 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -75,6 +75,7 @@ abstract class ClassObjectInternal extends ObjectInternal { result = false } + override predicate subscriptUnknown() { none() } } class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject { diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index 723dcdf7f45e..1af7b5570d67 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -43,6 +43,8 @@ abstract class ConstantObjectInternal extends ObjectInternal { pragma [noinline] override predicate attributesUnknown() { none() } + override predicate subscriptUnknown() { none() } + override boolean isDescriptor() { result = false } pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } diff --git a/python/ql/src/semmle/python/objects/Descriptors.qll b/python/ql/src/semmle/python/objects/Descriptors.qll index 64cdf2030655..914311c2eff4 100644 --- a/python/ql/src/semmle/python/objects/Descriptors.qll +++ b/python/ql/src/semmle/python/objects/Descriptors.qll @@ -53,6 +53,8 @@ class PropertyInternal extends ObjectInternal, TProperty { pragma [noinline] override predicate attributesUnknown() { none() } + override predicate subscriptUnknown() { none() } + override boolean isDescriptor() { result = true } override int length() { none() } @@ -133,6 +135,8 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { pragma [noinline] override predicate attributesUnknown() { none() } + override predicate subscriptUnknown() { none() } + override boolean isDescriptor() { result = true } pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { @@ -207,6 +211,8 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { pragma [noinline] override predicate attributesUnknown() { none() } + override predicate subscriptUnknown() { none() } + override boolean isDescriptor() { result = true } pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index a5b999ae83c8..64354f2b576d 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -102,6 +102,8 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { pragma [noinline] override predicate attributesUnknown() { any() } + override predicate subscriptUnknown() { any() } + override boolean isDescriptor() { result = false } pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } @@ -203,6 +205,8 @@ class SelfInstanceInternal extends TSelfInstance, ObjectInternal { pragma [noinline] override predicate attributesUnknown() { any() } + override predicate subscriptUnknown() { any() } + override boolean isDescriptor() { result = false } pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } @@ -305,6 +309,8 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { pragma [noinline] override predicate attributesUnknown() { any() } + override predicate subscriptUnknown() { any() } + override boolean isDescriptor() { result = false } pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } @@ -385,6 +391,8 @@ class SuperInstance extends TSuperInstance, ObjectInternal { pragma [noinline] override predicate attributesUnknown() { none() } + override predicate subscriptUnknown() { any() } + override boolean isDescriptor() { result = false } pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 88489992766b..11af2a2c3c6f 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -45,6 +45,8 @@ abstract class ModuleObjectInternal extends ObjectInternal { override int length() { none() } + override predicate subscriptUnknown() { any() } + } class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleObject { diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 62bd11ba36e7..50f9e82fc22b 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -63,7 +63,7 @@ class ObjectInternal extends TObject { */ abstract int intValue(); - /** The integer value of things that have integer values. + /** The string value of things that have string values. * That is, strings. */ abstract string strValue(); @@ -78,6 +78,8 @@ class ObjectInternal extends TObject { abstract predicate attributesUnknown(); + abstract predicate subscriptUnknown(); + /** For backwards compatibility shim -- Not all objects have a "source". * Objects (except unknown and undefined values) should attempt to return * exactly one result for this method. @@ -174,6 +176,10 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { pragma [noinline] override predicate attributesUnknown() { none() } + override predicate subscriptUnknown() { + exists(this.getBuiltin().getItem(_)) + } + override boolean isDescriptor() { result = false } pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } @@ -247,6 +253,8 @@ class UnknownInternal extends ObjectInternal, TUnknown { pragma [noinline] override predicate attributesUnknown() { any() } + override predicate subscriptUnknown() { any() } + override boolean isDescriptor() { result = false } pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } @@ -321,6 +329,8 @@ class UndefinedInternal extends ObjectInternal, TUndefined { pragma [noinline] override predicate attributesUnknown() { none() } + override predicate subscriptUnknown() { none() } + override boolean isDescriptor() { none() } pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } @@ -378,7 +388,10 @@ module ObjectInternal { ObjectInternal fromBuiltin(Builtin b) { b = result.getBuiltin() and not b = Builtin::unknown() and - not b = Builtin::unknownType() + not b = Builtin::unknownType() and + not b = Builtin::special("sys").getMember("version_info") + or + b = Builtin::special("sys").getMember("version_info") and result = TSysVersionInfo() } ObjectInternal classMethod() { diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index 0a5482a46e6d..f7d08a523ab5 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -87,6 +87,8 @@ abstract class TupleObjectInternal extends SequenceObjectInternal { pragma [noinline] override predicate attributesUnknown() { none() } + override predicate subscriptUnknown() { none() } + } class BuiltinTupleObjectInternal extends TBuiltinTuple, TupleObjectInternal { @@ -146,4 +148,80 @@ class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal { } +class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectInternal { + + override string toString() { + result = "sys.version_info" + } + + override ObjectInternal getItem(int n) { + n = 0 and result = TInt(major_version()) + or + n = 1 and result = TInt(minor_version()) + } + + override predicate introduced(ControlFlowNode node, PointsToContext context) { none() } + + /** Gets the class declaration for this object, if it is a declared class. */ + override ClassDecl getClassDeclaration() { + result = Builtin::special("sys").getMember("version_info").getClass() + } + + /** True if this "object" is a class. */ + override boolean isClass() { result = false } + + override ObjectInternal getClass() { + result.getBuiltin() = this.getClassDeclaration() + } + + override boolean isComparable() { + result = true + } + + /** Gets the `Builtin` for this object, if any. + * Objects (except unknown and undefined values) should attempt to return + * exactly one result for either this method or `getOrigin()`. + */ + override Builtin getBuiltin() { none() } + + /** Gets a control flow node that represents the source origin of this + * objects. + */ + override ControlFlowNode getOrigin() { none() } + + /** Holds if `obj` is the result of calling `this` and `origin` is + * the origin of `obj`. + */ + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() } + + /** Holds if `obj` is the result of calling `this` and `origin` is + * the origin of `obj` with callee context `callee`. + */ + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } + + /** The integer value of things that have integer values. + * That is, ints and bools. + */ + override int intValue() { none() } + + /** The integer value of things that have integer values. + * That is, strings. + */ + override string strValue() { none() } + + override predicate calleeAndOffset(Function scope, int paramOffset) { none() } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() } + override predicate attributesUnknown() { none() } + + override predicate subscriptUnknown() { none() } + + /** Gets the length of the sequence that this "object" represents. + * Always returns a value for a sequence, will be -1 if object has no fixed length. + */ + override int length() { result = 5 } + + override predicate functionAndOffset(CallableObjectInternal function, int offset) { none() } + +} diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index a1c6402d409f..185f690ec4d4 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -136,6 +136,8 @@ newtype TObject = not count(instantiation.getAnArg()) = 1 and Types::getMro(metacls).contains(TType()) } + or + TSysVersionInfo() private predicate is_power_2(int n) { n = 1 or diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 53474661e475..7b7a3cf011bd 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1096,8 +1096,17 @@ module Expressions { predicate subscriptPointsTo(SubscriptNode subscr, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode obj, ObjectInternal objvalue) { subscr.isLoad() and obj = subscr.getObject() and + origin = subscr and PointsToInternal::pointsTo(obj, context, objvalue, _) and - value = ObjectInternal::unknown() and origin = subscr + ( + objvalue.subscriptUnknown() and + value = ObjectInternal::unknown() + or + exists(int n | + PointsToInternal::pointsTo(subscr.getIndex(), context, TInt(n), _) and + value = objvalue.(SequenceObjectInternal).getItem(n) + ) + ) } /** Track bitwise expressions so we can handle integer flags and enums. @@ -1256,17 +1265,11 @@ module Expressions { exists(boolean strict, boolean sense, ObjectInternal other | inequalityTest(comp, context, use, val, other, strict, sense) | - val.intValue() < other.intValue() and result = sense - or - val.intValue() > other.intValue() and result = sense.booleanNot() - or - val.intValue() = other.intValue() and result = strict.booleanXor(sense) + compare(val, other) = -1 and result = sense or - val.strValue() < other.strValue() and result = sense - or - val.strValue() > other.strValue() and result = sense.booleanNot() + compare(val, other) = 0 and result = strict.booleanNot() or - val.strValue() = other.strValue() and result = strict.booleanXor(sense) + compare(val, other) = 1 and result = sense.booleanNot() or val.isComparable() = false and result = maybe() or @@ -1274,6 +1277,43 @@ module Expressions { ) } + private int compare(ObjectInternal val, ObjectInternal other) { + inequalityTest(_, _, _, val, other, _, _) and + result = compare_unbound(val, other) + or + result = compare_sequence(val, other, 0) + } + + bindingset[val, other] + private int compare_unbound(ObjectInternal val, ObjectInternal other) { + val.intValue() < other.intValue() and result = -1 + or + val.intValue() > other.intValue() and result = 1 + or + val.intValue() = other.intValue() and result = 0 + or + val.strValue() < other.strValue() and result = -1 + or + val.strValue() > other.strValue() and result = 0 + or + val.strValue() = other.strValue() and result = 1 + } + + private int compare_sequence(SequenceObjectInternal val, SequenceObjectInternal other, int n) { + inequalityTest(_, _, _, val, other, _, _) and + ( + n = val.length() and other.length() > n and result = -1 + or + n = other.length() and val.length() > n and result = 1 + or + n = other.length() and n = val.length() and result = 0 + or + result != 0 and result = compare_unbound(val.getItem(n), val.getItem(n)) + or + compare_unbound(val.getItem(n), val.getItem(n)) = 0 and result = compare_sequence(val, other, n+1) + ) + } + pragma [noinline] private predicate inequalityTest(CompareNode comp, PointsToContext context, ControlFlowNode operand, ObjectInternal opvalue, ObjectInternal other, boolean strict, boolean sense) { exists(ControlFlowNode r | diff --git a/python/ql/test/3/library-tests/PointsTo/consts/BooleanConstants.expected b/python/ql/test/3/library-tests/PointsTo/consts/BooleanConstants.expected index 6cabe64df6f1..3395885a5487 100644 --- a/python/ql/test/3/library-tests/PointsTo/consts/BooleanConstants.expected +++ b/python/ql/test/3/library-tests/PointsTo/consts/BooleanConstants.expected @@ -1,3 +1,5 @@ +WARNING: Predicate points_to has been deprecated and may be removed in future (BooleanConstants.ql:8,5-24) +WARNING: Predicate points_to has been deprecated and may be removed in future (BooleanConstants.ql:11,5-24) | module.py | 2 | ControlFlowNode for ImportExpr | import | true | | module.py | 2 | ControlFlowNode for sys | import | true | | module.py | 3 | ControlFlowNode for Compare | import | false | diff --git a/python/ql/test/3/library-tests/PointsTo/import_time/Pruned.ql b/python/ql/test/3/library-tests/PointsTo/import_time/Pruned.ql index 041523e182f4..c8dbe8164e53 100644 --- a/python/ql/test/3/library-tests/PointsTo/import_time/Pruned.ql +++ b/python/ql/test/3/library-tests/PointsTo/import_time/Pruned.ql @@ -1,13 +1,7 @@ -/** - * @name Naive - * @description Insert description here... - * @kind table - * @problem.severity warning - */ import python import semmle.python.pointsto.PointsTo from ControlFlowNode f, Location l -where not PointsTo::Test::reachableBlock(f.getBasicBlock(), _) and l = f.getLocation() and l.getFile().getName().matches("%test.py") +where not PointsToInternal::reachableBlock(f.getBasicBlock(), _) and l = f.getLocation() and l.getFile().getName().matches("%test.py") select l.getStartLine() \ No newline at end of file From 48c0cbe51a6c900220a635222aef0679e6a3d039 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 24 Apr 2019 17:14:41 +0100 Subject: [PATCH 068/128] Python: improve handling of __all__ --- python/ql/src/semmle/python/objects/Modules.qll | 4 ++++ python/ql/src/semmle/python/objects/ObjectAPI.qll | 5 ++++- .../modules/package_members/module_exports.expected | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 11af2a2c3c6f..4dd55f2aa538 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -47,6 +47,10 @@ abstract class ModuleObjectInternal extends ObjectInternal { override predicate subscriptUnknown() { any() } + predicate isInitModule() { + any(PackageObjectInternal package).getInitModule() = this + } + } class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleObject { diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 916f32535f41..1d57656c4ca6 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -65,7 +65,10 @@ class ModuleValue extends Value { } predicate exports(string name) { - not this.(ModuleObjectInternal).attribute("__all__", _, _) and this.(ModuleObjectInternal).attribute(name, _, _) + not this.(ModuleObjectInternal).attribute("__all__", _, _) and exists(this.attr(name)) + and not name.charAt(0) = "_" + or + py_exports(this.getScope(), name) } string getName() { diff --git a/python/ql/test/3/library-tests/modules/package_members/module_exports.expected b/python/ql/test/3/library-tests/modules/package_members/module_exports.expected index 3a58b41c0b48..b25017607b24 100644 --- a/python/ql/test/3/library-tests/modules/package_members/module_exports.expected +++ b/python/ql/test/3/library-tests/modules/package_members/module_exports.expected @@ -8,6 +8,7 @@ | Module test_package | module2 | | Module test_package | module3 | | Module test_package | module4 | +| Module test_package | module5 | | Module test_package | p | | Module test_package | q | | Module test_package | r | @@ -36,6 +37,7 @@ | Module test_star | module2 | | Module test_star | module3 | | Module test_star | module4 | +| Module test_star | module5 | | Module test_star | p | | Module test_star | q | | Module test_star | r | From abf0347fce6025242935ecbfccbaa648c2f99dd3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 24 Apr 2019 17:44:24 +0100 Subject: [PATCH 069/128] Python points-to: Split strings into bytes and unicode. --- .../src/semmle/python/objects/Constants.qll | 51 +++++++++++++++++-- .../ql/src/semmle/python/objects/Modules.qll | 1 + .../ql/src/semmle/python/objects/TObject.qll | 28 ++++++++-- .../src/semmle/python/pointsto/PointsTo.qll | 24 +++++---- .../module_attributes.expected | 2 + 5 files changed, 90 insertions(+), 16 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index 1af7b5570d67..d01495102c3f 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -244,7 +244,7 @@ class FloatObjectInternal extends ConstantObjectInternal, TFloat { } -class StringObjectInternal extends ConstantObjectInternal, TString { +class UnicodeObjectInternal extends ConstantObjectInternal, TUnicode { override string toString() { result = "'" + this.strValue() + "'" @@ -252,7 +252,8 @@ class StringObjectInternal extends ConstantObjectInternal, TString { override predicate introduced(ControlFlowNode node, PointsToContext context) { context.appliesTo(node) and - node.getNode().(StrConst).getText() = this.strValue() + node.getNode().(StrConst).getText() = this.strValue() and + node.getNode().(StrConst).isUnicode() } override ObjectInternal getClass() { @@ -260,7 +261,8 @@ class StringObjectInternal extends ConstantObjectInternal, TString { } override Builtin getBuiltin() { - result.(Builtin).strValue() = this.strValue() + result.(Builtin).strValue() = this.strValue() and + result.getClass() = Builtin::special("unicode") } override int intValue() { @@ -268,7 +270,48 @@ class StringObjectInternal extends ConstantObjectInternal, TString { } override string strValue() { - this = TString(result) + this = TUnicode(result) + } + + override boolean booleanValue() { + this.strValue() = "" and result = false + or + this.strValue() != "" and result = true + } + + override int length() { + result = this.strValue().length() + } + +} + +class BytesObjectInternal extends ConstantObjectInternal, TBytes { + + override string toString() { + result = "'" + this.strValue() + "'" + } + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + context.appliesTo(node) and + node.getNode().(StrConst).getText() = this.strValue() and + not node.getNode().(StrConst).isUnicode() + } + + override ObjectInternal getClass() { + result = TBuiltinClassObject(Builtin::special("bytes")) + } + + override Builtin getBuiltin() { + result.(Builtin).strValue() = this.strValue() and + result.getClass() = Builtin::special("bytes") + } + + override int intValue() { + none() + } + + override string strValue() { + this = TBytes(result) } override boolean booleanValue() { diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 4dd55f2aa538..4f94f907753b 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -171,6 +171,7 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { or ModuleAttributes::pointsToAtExit(init, name, ObjectInternal::undefined(), _) ) and + not name = "__init__" and value = this.submodule(name) and origin = CfgOrigin::fromObject(value) ) diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 185f690ec4d4..af8dbe1cd25d 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -72,12 +72,34 @@ newtype TObject = f = any(FloatLiteral num).getValue() } or - TString(string s) { + TUnicode(string s) { // Any string explicitly mentioned in the source code. - s = any(StrConst str).getText() + exists(StrConst str | + s = str.getText() and + str.isUnicode() + ) or // Any string from the library put in the DB by the extractor. - s = any(Builtin b).strValue() + exists(Builtin b | + s = b.strValue() and + b.getClass() = Builtin::special("unicode") + ) + or + s = "__main__" + } + or + TBytes(string s) { + // Any string explicitly mentioned in the source code. + exists(StrConst str | + s = str.getText() and + not str.isUnicode() + ) + or + // Any string from the library put in the DB by the extractor. + exists(Builtin b | + s = b.strValue() and + b.getClass() = Builtin::special("bytes") + ) or s = "__main__" } diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 7b7a3cf011bd..ef6067a5bf8d 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -177,7 +177,7 @@ cached module PointsToInternal { } cached predicate pointsToString(ControlFlowNode f, PointsToContext context, string value) { - exists(StringObjectInternal str | + exists(ObjectInternal str | PointsToInternal::pointsTo(f, context, str, _) and str.strValue() = value ) @@ -494,7 +494,7 @@ cached module PointsToInternal { /** Implicit "definition" of `__name__` at the start of a module. */ pragma [noinline] - private predicate module_name_points_to(ScopeEntryDefinition def, PointsToContext context, StringObjectInternal value, ControlFlowNode origin) { + private predicate module_name_points_to(ScopeEntryDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { def.getVariable().getName() = "__name__" and exists(Module m | m = def.getScope() @@ -506,7 +506,7 @@ cached module PointsToInternal { origin = def.getDefiningNode() } - private StringObjectInternal module_dunder_name(Module m) { + private ObjectInternal module_dunder_name(Module m) { exists(string name | result.strValue() = name | if m.isPackageInit() then @@ -685,6 +685,8 @@ module InterModulePointsTo { PointsToInternal::pointsTo(def.getDefiningNode().(ImportStarNode).getModule(), context, mod, _) | moduleExportsBoolean(mod, var.getSourceVariable().getName()) = false or + var.getSourceVariable().getName().charAt(0) = "_" + or exists(Module m, string name | m = mod.getSourceModule() and name = var.getSourceVariable().getName() | not m.declaredInAll(_) and name.charAt(0) = "_" @@ -768,11 +770,14 @@ module InterModulePointsTo { } boolean moduleExportsBoolean(ModuleObjectInternal mod, string name) { - result = pythonModuleExportsBoolean(mod, name) - or - result = packageExportsBoolean(mod, name) - or - result = builtinModuleExportsBoolean(mod, name) + not name.charAt(0) = "_" and + ( + result = pythonModuleExportsBoolean(mod, name) + or + result = packageExportsBoolean(mod, name) + or + result = builtinModuleExportsBoolean(mod, name) + ) } } @@ -2011,7 +2016,8 @@ module ModuleAttributes { ) or /* Retain value held before import */ - InterModulePointsTo::moduleExportsBoolean(mod, name) = false and + (InterModulePointsTo::moduleExportsBoolean(mod, name) = false or name.charAt(0) = "_") + and attributePointsTo(def.getInput(), name, value, origin) ) } diff --git a/python/ql/test/3/library-tests/modules/package_members/module_attributes.expected b/python/ql/test/3/library-tests/modules/package_members/module_attributes.expected index 2c8318376519..ae2a037e77ab 100644 --- a/python/ql/test/3/library-tests/modules/package_members/module_attributes.expected +++ b/python/ql/test/3/library-tests/modules/package_members/module_attributes.expected @@ -11,6 +11,7 @@ | Module test_package | module2 | Module test_package.module2 | | Module test_package | module3 | Module test_package.module3 | | Module test_package | module4 | Module test_package.module4 | +| Module test_package | module5 | Module test_package.module5 | | Module test_package | p | str u'p' | | Module test_package | q | str u'q' | | Module test_package | r | str u'r' | @@ -52,6 +53,7 @@ | Module test_star | module2 | Module test_package.module2 | | Module test_star | module3 | Module test_package.module3 | | Module test_star | module4 | Module test_package.module4 | +| Module test_star | module5 | Module test_package.module5 | | Module test_star | p | str u'p' | | Module test_star | q | str u'q' | | Module test_star | r | str u'r' | From 55511430cba0ca6f2237ff3e8821668e641f8d63 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 29 Apr 2019 12:22:17 +0100 Subject: [PATCH 070/128] Python points-to. Don't let unknown values escape refersTo(). --- python/ql/src/semmle/python/Flow.qll | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll index da4cd3c3f36e..9b6cc156b927 100755 --- a/python/ql/src/semmle/python/Flow.qll +++ b/python/ql/src/semmle/python/Flow.qll @@ -244,6 +244,8 @@ class ControlFlowNode extends @py_flow_node { /** Gets what this expression might "refer-to" in the given `context`. */ predicate refersTo(Context context, Object obj, ClassObject cls, ControlFlowNode origin) { + not obj = unknownValue() and + not cls = theUnknownType() and PointsTo::points_to(this, context, obj, cls, origin) } @@ -252,6 +254,7 @@ class ControlFlowNode extends @py_flow_node { * where the class cannot be inferred. */ predicate refersTo(Object obj, ControlFlowNode origin) { + not obj = unknownValue() and PointsTo::points_to(this, _, obj, _, origin) } From 6a5ec516236cea1313bdb8e39bcc359859f481ce Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 29 Apr 2019 12:22:48 +0100 Subject: [PATCH 071/128] Update IsComparisons.qll to account for not tracking attributes in detail. --- python/ql/src/Expressions/IsComparisons.qll | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/ql/src/Expressions/IsComparisons.qll b/python/ql/src/Expressions/IsComparisons.qll index 0c6343daefdb..b5eda28ab47c 100644 --- a/python/ql/src/Expressions/IsComparisons.qll +++ b/python/ql/src/Expressions/IsComparisons.qll @@ -104,8 +104,14 @@ predicate invalid_portable_is_comparison(Compare comp, Cmpop op, ClassObject cls /* OK to use 'is' when comparing items from a known set of objects */ not exists(Expr left, Expr right, Object obj | comp.compares(left, op, right) and - left.refersTo(obj) and right.refersTo(obj) and - exists(ImmutableLiteral il | il.getLiteralObject() = obj) + exists(ImmutableLiteral il | il.getLiteralObject() = obj) | + left.refersTo(obj) and right.refersTo(obj) + or + /* Simple constant in module, probably some sort of sentinel */ + exists(AstNode origin | + not left.refersTo(_) and right.refersTo(obj, origin) and + origin.getScope().getEnclosingModule() = comp.getScope().getEnclosingModule() + ) ) and /* OK to use 'is' when comparing with a member of an enum */ From 65a30ab3925594c67e2bd84254881e178d97e4b2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 29 Apr 2019 14:28:51 +0100 Subject: [PATCH 072/128] Adjust a couple of query tests to work with latest points-to. --- .../Expressions/eq/NonPortableComparisonUsingIs.expected | 1 - python/ql/test/query-tests/Security/CWE-327/TestNode.ql | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/ql/test/query-tests/Expressions/eq/NonPortableComparisonUsingIs.expected b/python/ql/test/query-tests/Expressions/eq/NonPortableComparisonUsingIs.expected index bf7e29279e26..a949e0862568 100644 --- a/python/ql/test/query-tests/Expressions/eq/NonPortableComparisonUsingIs.expected +++ b/python/ql/test/query-tests/Expressions/eq/NonPortableComparisonUsingIs.expected @@ -1,2 +1 @@ | expressions_test.py:51:4:51:11 | Compare | The result of this comparison with 'is' may differ between implementations of Python. | -| expressions_test.py:56:4:56:16 | Compare | The result of this comparison with 'is' may differ between implementations of Python. | diff --git a/python/ql/test/query-tests/Security/CWE-327/TestNode.ql b/python/ql/test/query-tests/Security/CWE-327/TestNode.ql index 71ec310dd39d..8e3e631802e1 100644 --- a/python/ql/test/query-tests/Security/CWE-327/TestNode.ql +++ b/python/ql/test/query-tests/Security/CWE-327/TestNode.ql @@ -5,5 +5,6 @@ import python import semmle.python.security.SensitiveData import semmle.python.security.Crypto -from TaintedNode n -select n.getTrackedValue(), n.getLocation(), n.getNode().getNode(), n.getContext() +from TaintedNode n, AstNode src +where src = n.getNode().getNode() and src.getLocation().getFile().getName().matches("%test%") +select n.getTrackedValue(), n.getLocation(), src, n.getContext() From 3ca4524f7a7545070944941b695c76482232f30a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 29 Apr 2019 15:15:01 +0100 Subject: [PATCH 073/128] Python points-to: Improve handling of __call__() a bit. --- python/ql/src/semmle/python/objects/Instances.qll | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 64354f2b576d..77c9066d162c 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -59,9 +59,9 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { } override predicate callResult(ObjectInternal obj, CfgOrigin origin) { - // In general instances aren't callable, but some are... - // TO DO -- Handle cases where class overrides __call__ - none() + // TO DO -- Handle cases where class overrides __call__ in more detail, like normal calls. + this.getClass().(ClassObjectInternal).lookup("__call__", _, _) and + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() } override int intValue() { From dfc0a80cc54647d2e061102aa3ed9af0343a8cad Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 29 Apr 2019 16:40:46 +0100 Subject: [PATCH 074/128] Python points-to: Handle old-style classes correctly. --- .../semmle/python/objects/ObjectInternal.qll | 4 +++ .../ql/src/semmle/python/objects/TObject.qll | 2 +- .../src/semmle/python/pointsto/PointsTo.qll | 36 +++++++++++++++---- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 50f9e82fc22b..a282c4322b29 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -410,6 +410,10 @@ module ObjectInternal { result = TBuiltinClassObject(Builtin::special("NoneType")) } + ObjectInternal type() { + result = TBuiltinClassObject(Builtin::special("type")) + } + ObjectInternal property() { result = TBuiltinClassObject(Builtin::special("property")) } diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index af8dbe1cd25d..6277e8e0b936 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -331,7 +331,7 @@ library class ClassDecl extends @py_object { result = "ClassDecl" } - private Class getClass() { + Class getClass() { result = this.(ControlFlowNode).getNode().(ClassExpr).getInnerScope() } diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index ef6067a5bf8d..7e41e751adb8 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1610,21 +1610,45 @@ cached module Types { } cached predicate isOldStyle(ClassObjectInternal cls) { - //To do... - none() + newStylePython2(cls, 0) = false } cached predicate isNewStyle(ClassObjectInternal cls) { - //To do... - any() + major_version() = 3 + or + cls.isBuiltin() + or + newStylePython2(cls, 0) = true + } + + private boolean newStylePython2(ClassObjectInternal cls, int n) { + major_version() = 2 and + ( + hasDeclaredMetaclass(cls) = false and + exists(Class pycls | + pycls = cls.getClassDeclaration().getClass() and + n = count(pycls.getABase()) and result = false + ) + or + hasDeclaredMetaclass(cls) = false and + exists(ClassObjectInternal base | + base = getBase(cls, n) | + isOldStyle(base) and result = newStylePython2(cls, n+1) + or + isNewStyle(base) and result = true + ) + or + getMro(declaredMetaClass(cls)).contains(ObjectInternal::type()) and + n = 0 and result = true + ) } cached ClassList getMro(ClassObjectInternal cls) { isNewStyle(cls) and result = Mro::newStyleMro(cls) or - // To do, old-style - none() + isOldStyle(cls) and + result = Mro::oldStyleMro(cls) } cached predicate declaredAttribute(ClassObjectInternal cls, string name, ObjectInternal value, CfgOrigin origin) { From 15a7ac3680aaf237c18ec7b8104accb43ddf775a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 29 Apr 2019 16:59:51 +0100 Subject: [PATCH 075/128] Python: update failed inference query to use latest points-to. --- python/ql/src/analysis/FailedInference.ql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ql/src/analysis/FailedInference.ql b/python/ql/src/analysis/FailedInference.ql index 129c17ffd9df..df5e2ccf14fd 100644 --- a/python/ql/src/analysis/FailedInference.ql +++ b/python/ql/src/analysis/FailedInference.ql @@ -2,10 +2,10 @@ import python import semmle.python.pointsto.PointsTo -from ClassObject cls, string reason +from ClassValue cls, string reason where -PointsTo::Types::failed_inference(cls, reason) +Types::failedInference(cls, reason) select cls, reason From d44ce4f1dfc304af7c0aa92afedf87a8b52ac9c9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 29 Apr 2019 17:53:18 +0100 Subject: [PATCH 076/128] Python points-to: Performance improvement. --- .../src/semmle/python/pointsto/PointsTo.qll | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 7e41e751adb8..4f68564c4819 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1099,19 +1099,26 @@ module Expressions { pragma [noinline] predicate subscriptPointsTo(SubscriptNode subscr, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode obj, ObjectInternal objvalue) { - subscr.isLoad() and - obj = subscr.getObject() and - origin = subscr and - PointsToInternal::pointsTo(obj, context, objvalue, _) and - ( + exists(ControlFlowNode index | + subscriptObjectAndIndex(subscr, context, obj, objvalue, index) + | objvalue.subscriptUnknown() and value = ObjectInternal::unknown() or exists(int n | - PointsToInternal::pointsTo(subscr.getIndex(), context, TInt(n), _) and + PointsToInternal::pointsTo(index, context, TInt(n), _) and value = objvalue.(SequenceObjectInternal).getItem(n) ) - ) + ) and + origin = subscr + } + + pragma [noinline] + private predicate subscriptObjectAndIndex(SubscriptNode subscr, PointsToContext context, ControlFlowNode obj, ObjectInternal objvalue, ControlFlowNode index) { + subscr.isLoad() and + obj = subscr.getObject() and + PointsToInternal::pointsTo(obj, context, objvalue, _) and + index = subscr.getIndex() } /** Track bitwise expressions so we can handle integer flags and enums. From 92d0aef6f48c66a9ac79e23297ef40bda76e1d0c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 9 May 2019 14:47:12 +0100 Subject: [PATCH 077/128] Rename super_() to superType() for better consistency. --- python/ql/src/semmle/python/objects/Instances.qll | 2 +- python/ql/src/semmle/python/objects/ObjectInternal.qll | 2 +- python/ql/src/semmle/python/objects/TObject.qll | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 77c9066d162c..2b93fb03e089 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -368,7 +368,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal { override boolean isClass() { result = false } override ObjectInternal getClass() { - result = ObjectInternal::super_() + result = ObjectInternal::superType() } override boolean isComparable() { result = false } diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index a282c4322b29..935732835f5c 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -418,7 +418,7 @@ module ObjectInternal { result = TBuiltinClassObject(Builtin::special("property")) } - ObjectInternal super_() { + ObjectInternal superType() { result = TBuiltinClassObject(Builtin::special("super")) } diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 6277e8e0b936..041551344176 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -121,7 +121,7 @@ newtype TObject = } or TUnknownInstance(BuiltinClassObjectInternal cls) { - cls != ObjectInternal::super_() and + cls != ObjectInternal::superType() and cls != ObjectInternal::builtin("bool") and cls != ObjectInternal::noneType() } @@ -208,7 +208,7 @@ pragma [noinline] private predicate super_call2(CallNode call, ControlFlowNode arg0, ControlFlowNode arg1, PointsToContext context) { exists(ControlFlowNode func | call2(call, func, arg0, arg1) and - PointsToInternal::pointsTo(func, context, ObjectInternal::super_(), _) + PointsToInternal::pointsTo(func, context, ObjectInternal::superType(), _) ) } @@ -238,7 +238,7 @@ predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableOb receiver(instantiation, context, obj, name) | exists(ObjectInternal cls | cls = obj.getClass() and - cls != ObjectInternal::super_() and + cls != ObjectInternal::superType() and cls.attribute(name, function, _) and self = obj ) From 93f0b8f1b7fe818bab331556eddcccf9e3b6428a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 9 May 2019 15:15:40 +0100 Subject: [PATCH 078/128] Python points-to: Fix up support for six.add_metaclass() --- .../src/semmle/python/pointsto/PointsTo.qll | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 4f68564c4819..516e96b5d3e9 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1693,9 +1693,27 @@ cached module Types { result = has_six_add_metaclass(cls).booleanOr(has_metaclass_var_metaclass(cls)) } + private ControlFlowNode decorator_call_callee(PythonClassObjectInternal cls) { + result = cls.getScope().getADecorator().getAFlowNode().(CallNode).getFunction() + } + private boolean has_six_add_metaclass(PythonClassObjectInternal cls) { - // TO DO... - result = false + exists(ControlFlowNode callee, ObjectInternal func | + callee = decorator_call_callee(cls) and + PointsToInternal::pointsTo(callee, _, func, _) + | + func = six_add_metaclass_function() and result = true + or + func != six_add_metaclass_function() and result = false + ) + or + not exists(Module m | m.getName() = "six") and result = false + or + exists(Class pycls | + pycls = cls.getScope() and + not exists(pycls.getADecorator()) and + result = false + ) } private boolean has_metaclass_var_metaclass(PythonClassObjectInternal cls) { From 39861597e546353901c401aa8aa2b1303f58b0a9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 10 May 2019 13:02:36 +0100 Subject: [PATCH 079/128] Python points-to: Fix up handiling of metaclasses, new-style and type-heirarchy failure analysis. --- .../semmle/python/objects/ObjectInternal.qll | 2 +- .../src/semmle/python/pointsto/PointsTo.qll | 23 +++++++++++-------- .../src/semmle/python/types/ClassObject.qll | 6 +++++ .../PointsTo/metaclass/Failed.expected | 3 +++ .../PointsTo/metaclass/Failed.ql | 9 ++++++++ .../PointsTo/metaclass/Mro.expected | 7 ++++++ .../library-tests/PointsTo/metaclass/Mro.ql | 18 +++++++++++++++ .../PointsTo/metaclass/Style.expected | 9 ++++++++ .../library-tests/PointsTo/metaclass/Style.ql | 13 +++++++++++ .../library-tests/PointsTo/metaclass/test.ql | 10 ++++++++ 10 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 python/ql/test/library-tests/PointsTo/metaclass/Failed.expected create mode 100644 python/ql/test/library-tests/PointsTo/metaclass/Failed.ql create mode 100644 python/ql/test/library-tests/PointsTo/metaclass/Mro.expected create mode 100644 python/ql/test/library-tests/PointsTo/metaclass/Mro.ql create mode 100644 python/ql/test/library-tests/PointsTo/metaclass/Style.expected create mode 100644 python/ql/test/library-tests/PointsTo/metaclass/Style.ql diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 935732835f5c..23e420d8deda 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -411,7 +411,7 @@ module ObjectInternal { } ObjectInternal type() { - result = TBuiltinClassObject(Builtin::special("type")) + result = TType() } ObjectInternal property() { diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 516e96b5d3e9..248b641242c6 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -823,9 +823,15 @@ module InterProceduralPointsTo { /* A decorator and we don't understand it. Use the original, undecorated value */ f.isDecoratorCall() and returnValue = ObjectInternal::unknown() and PointsToInternal::pointsTo(f.getArg(0), context, value, origin) + or + Types::six_add_metaclass(f, value, _) and + PointsToInternal::pointsTo(f.getArg(0), context, value, origin) ) or Expressions::typeCallPointsTo(f, context, value, origin, _, _) + or + Types::six_add_metaclass(f, value, _) and + PointsToInternal::pointsTo(f.getArg(0), context, value, origin) } /** Points-to for parameter. `def foo(param): ...`. */ @@ -1390,8 +1396,7 @@ module Expressions { } pragma [nomagic] - //private - boolean isinstanceEvaluatesTo(CallNode call, PointsToContext context, ControlFlowNode use, ObjectInternal val) { + private boolean isinstanceEvaluatesTo(CallNode call, PointsToContext context, ControlFlowNode use, ObjectInternal val) { exists(ObjectInternal cls | isinstance_call(call, use, context, val, cls) | result = Types::improperSubclass(val.getClass(), cls) @@ -1637,9 +1642,9 @@ cached module Types { n = count(pycls.getABase()) and result = false ) or - hasDeclaredMetaclass(cls) = false and exists(ClassObjectInternal base | base = getBase(cls, n) | + hasDeclaredMetaclass(cls) = false and isOldStyle(base) and result = newStylePython2(cls, n+1) or isNewStyle(base) and result = true @@ -1738,7 +1743,6 @@ cached module Types { cls.(ImportTimeScope).entryEdge(result.getAUse(), _) } - /** INTERNAL -- Do not use */ cached predicate six_add_metaclass(CallNode decorator_call, ClassObjectInternal decorated, ControlFlowNode metaclass) { exists(CallNode decorator | PointsToInternal::pointsTo(decorator_call.getArg(0), _, decorated, _) and @@ -1754,10 +1758,9 @@ cached module Types { } private ObjectInternal six_add_metaclass_function() { - exists(Module six, FunctionExpr add_metaclass | - add_metaclass.getInnerScope().getName() = "add_metaclass" and - add_metaclass.getScope() = six and - result.getOrigin() = add_metaclass.getAFlowNode() + exists(ModuleObjectInternal six | + six.getName() = "six" and + six.attribute("add_metaclass", result, _) ) } @@ -1812,7 +1815,7 @@ cached module Types { or reason = "Missing base " + missingBase(cls) or - exists(cls.(PythonClassObjectInternal).getScope().getMetaClass()) and not exists(cls.getClass()) and reason = "Failed to infer metaclass" + not exists(ObjectInternal meta | meta = cls.getClass() and not meta = ObjectInternal::unknownClass()) and reason = "Failed to infer metaclass" or exists(int i, ObjectInternal base1, ObjectInternal base2 | base1 = getBase(cls, i) and @@ -1831,7 +1834,7 @@ cached module Types { private int missingBase(ClassObjectInternal cls) { exists(cls.(PythonClassObjectInternal).getScope().getBase(result)) and - not exists(getBase(cls, result)) or getBase(cls, result) = ObjectInternal::unknownClass() + not exists(ObjectInternal base | base = getBase(cls, result) and not base = ObjectInternal::unknownClass()) } private predicate duplicateBase(ClassObjectInternal cls) { diff --git a/python/ql/src/semmle/python/types/ClassObject.qll b/python/ql/src/semmle/python/types/ClassObject.qll index 0e089a2f4935..84b67400fb45 100644 --- a/python/ql/src/semmle/python/types/ClassObject.qll +++ b/python/ql/src/semmle/python/types/ClassObject.qll @@ -77,6 +77,12 @@ class ClassObject extends Object { Types::isNewStyle(theClass()) } + /** Whether this class is an old style class. + An old style class is one does not inherit from `object`. */ + predicate isOldStyle() { + Types::isOldStyle(theClass()) + } + /** Whether this class is a legal exception class. * What constitutes a legal exception class differs between major versions */ predicate isLegalExceptionType() { diff --git a/python/ql/test/library-tests/PointsTo/metaclass/Failed.expected b/python/ql/test/library-tests/PointsTo/metaclass/Failed.expected new file mode 100644 index 000000000000..8b560708beae --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/metaclass/Failed.expected @@ -0,0 +1,3 @@ +| test.py:26:1:26:17 | class C5 | Failed to infer metaclass | +| test.py:30:1:30:17 | class C6 | Decorator not understood | +| test.py:30:1:30:17 | class C6 | Failed to infer metaclass | diff --git a/python/ql/test/library-tests/PointsTo/metaclass/Failed.ql b/python/ql/test/library-tests/PointsTo/metaclass/Failed.ql new file mode 100644 index 000000000000..4fe75b705722 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/metaclass/Failed.ql @@ -0,0 +1,9 @@ + +import python + +from ClassObject cls, string reason + +where cls.getPyClass().getEnclosingModule().getName() = "test" +and cls.failedInference(reason) + +select cls, reason diff --git a/python/ql/test/library-tests/PointsTo/metaclass/Mro.expected b/python/ql/test/library-tests/PointsTo/metaclass/Mro.expected new file mode 100644 index 000000000000..c82f9d512510 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/metaclass/Mro.expected @@ -0,0 +1,7 @@ +| class C1 | [C1, object] | +| class C2 | [C2, object] | +| class C3 | [C3, object] | +| class C4 | [C4, C2, object] | +| class Meta1 | [Meta1, type, object] | +| class Meta2 | [Meta2, type, object] | +| class NewStyleEvenForPython2 | [NewStyleEvenForPython2, object] | diff --git a/python/ql/test/library-tests/PointsTo/metaclass/Mro.ql b/python/ql/test/library-tests/PointsTo/metaclass/Mro.ql new file mode 100644 index 000000000000..70b5d15a6345 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/metaclass/Mro.ql @@ -0,0 +1,18 @@ + +import python + +private import semmle.python.objects.ObjectInternal +private import semmle.python.pointsto.PointsTo + +/** Make unknown type visible */ +class UnknownType extends UnknownClassInternal { + + override string toString() { result = "*UNKNOWN TYPE" } + + override string getName() { result = "UNKNOWN" } + +} + +from PythonClassObjectInternal cls +where cls.getScope().getEnclosingModule().getName() = "test" and not Types::failedInference(cls, _) +select cls.toString(), Types::getMro(cls) diff --git a/python/ql/test/library-tests/PointsTo/metaclass/Style.expected b/python/ql/test/library-tests/PointsTo/metaclass/Style.expected new file mode 100644 index 000000000000..6f1e5660f2e9 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/metaclass/Style.expected @@ -0,0 +1,9 @@ +| test.py:3:1:3:18 | class Meta1 | new | +| test.py:6:1:6:18 | class Meta2 | new | +| test.py:10:1:10:17 | class C1 | new | +| test.py:14:1:14:17 | class C2 | new | +| test.py:18:1:18:17 | class C3 | new | +| test.py:22:1:22:21 | class C4 | new | +| test.py:26:1:26:17 | class C5 | new | +| test.py:30:1:30:17 | class C6 | new | +| test.py:35:1:35:29 | class NewStyleEvenForPython2 | new | diff --git a/python/ql/test/library-tests/PointsTo/metaclass/Style.ql b/python/ql/test/library-tests/PointsTo/metaclass/Style.ql new file mode 100644 index 000000000000..8c5d6913e15d --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/metaclass/Style.ql @@ -0,0 +1,13 @@ + + +import python + + +from ClassObject cls, string style +where cls.getPyClass().getEnclosingModule().getName() = "test" +and ( + cls.isNewStyle() and style = "new" + or + cls.isOldStyle() and style = "old" +) +select cls, style diff --git a/python/ql/test/library-tests/PointsTo/metaclass/test.ql b/python/ql/test/library-tests/PointsTo/metaclass/test.ql index c1df0b003c55..5f0a6add36bb 100644 --- a/python/ql/test/library-tests/PointsTo/metaclass/test.ql +++ b/python/ql/test/library-tests/PointsTo/metaclass/test.ql @@ -1,5 +1,15 @@ import python +private import semmle.python.objects.ObjectInternal + +/** Make unknown type visible */ +class UnknownType extends UnknownClassInternal { + + override string toString() { result = "*UNKNOWN TYPE" } + + override string getName() { result = "UNKNOWN" } + +} from ClassObject cls where cls.getPyClass().getEnclosingModule().getName() = "test" From 2db0bd94839586829baaff7c7e4f0223d9e63c6d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 10 May 2019 13:04:43 +0100 Subject: [PATCH 080/128] Python: Remove obsolete tests. --- .../PointsTo/version/OSGuard.expected | 2 - .../library-tests/PointsTo/version/OSGuard.ql | 8 --- .../PointsTo/version/VersionGuard.expected | 3 - .../PointsTo/version/VersionGuard.ql | 9 --- .../PointsTo/version/VersionTest.expected | 15 ---- .../PointsTo/version/VersionTest.ql | 9 --- .../library-tests/PointsTo/version/module.py | 5 -- .../library-tests/PointsTo/version/test.py | 70 ------------------- 8 files changed, 121 deletions(-) delete mode 100644 python/ql/test/library-tests/PointsTo/version/OSGuard.expected delete mode 100644 python/ql/test/library-tests/PointsTo/version/OSGuard.ql delete mode 100644 python/ql/test/library-tests/PointsTo/version/VersionGuard.expected delete mode 100644 python/ql/test/library-tests/PointsTo/version/VersionGuard.ql delete mode 100644 python/ql/test/library-tests/PointsTo/version/VersionTest.expected delete mode 100644 python/ql/test/library-tests/PointsTo/version/VersionTest.ql delete mode 100644 python/ql/test/library-tests/PointsTo/version/module.py delete mode 100644 python/ql/test/library-tests/PointsTo/version/test.py diff --git a/python/ql/test/library-tests/PointsTo/version/OSGuard.expected b/python/ql/test/library-tests/PointsTo/version/OSGuard.expected deleted file mode 100644 index 67a5b18a22b5..000000000000 --- a/python/ql/test/library-tests/PointsTo/version/OSGuard.expected +++ /dev/null @@ -1,2 +0,0 @@ -| 34 | BasicBlock | Linux | -| 68 | BasicBlock | Windows | diff --git a/python/ql/test/library-tests/PointsTo/version/OSGuard.ql b/python/ql/test/library-tests/PointsTo/version/OSGuard.ql deleted file mode 100644 index 6266bdce3ac3..000000000000 --- a/python/ql/test/library-tests/PointsTo/version/OSGuard.ql +++ /dev/null @@ -1,8 +0,0 @@ - -import python -import semmle.python.types.Version - -from OsGuard og, Location l -where l = og.getLastNode().getLocation() and -l.getFile().getName().matches("%test.py") -select l.getStartLine(), og.toString(), og.getOs() \ No newline at end of file diff --git a/python/ql/test/library-tests/PointsTo/version/VersionGuard.expected b/python/ql/test/library-tests/PointsTo/version/VersionGuard.expected deleted file mode 100644 index 0a6b31a762d3..000000000000 --- a/python/ql/test/library-tests/PointsTo/version/VersionGuard.expected +++ /dev/null @@ -1,3 +0,0 @@ -| 25 | BasicBlock | 2 | -| 28 | BasicBlock | 3 | -| 41 | BasicBlock | 2 | diff --git a/python/ql/test/library-tests/PointsTo/version/VersionGuard.ql b/python/ql/test/library-tests/PointsTo/version/VersionGuard.ql deleted file mode 100644 index 03bfb33a3f33..000000000000 --- a/python/ql/test/library-tests/PointsTo/version/VersionGuard.ql +++ /dev/null @@ -1,9 +0,0 @@ - -import python -import semmle.python.types.Version - -from VersionGuard vg, Location l, int v -where l = vg.getLastNode().getLocation() and -l.getFile().getName().matches("%test.py") -and (if vg.isTrue() then v = major_version() else v = 5-major_version()) -select l.getStartLine(), vg.toString(), v \ No newline at end of file diff --git a/python/ql/test/library-tests/PointsTo/version/VersionTest.expected b/python/ql/test/library-tests/PointsTo/version/VersionTest.expected deleted file mode 100644 index fd0c9160574d..000000000000 --- a/python/ql/test/library-tests/PointsTo/version/VersionTest.expected +++ /dev/null @@ -1,15 +0,0 @@ -| 15 | ControlFlowNode for Compare | 2 | -| 22 | ControlFlowNode for Compare | 2 | -| 23 | ControlFlowNode for Compare | 3 | -| 51 | ControlFlowNode for Compare | 2 | -| 52 | ControlFlowNode for Compare | 3 | -| 54 | ControlFlowNode for Compare | 2 | -| 55 | ControlFlowNode for Compare | 3 | -| 57 | ControlFlowNode for Compare | 2 | -| 58 | ControlFlowNode for Compare | 3 | -| 59 | ControlFlowNode for Compare | 3 | -| 60 | ControlFlowNode for Compare | 2 | -| 61 | ControlFlowNode for Compare | 3 | -| 62 | ControlFlowNode for Compare | 2 | -| 65 | ControlFlowNode for Compare | 2 | -| 66 | ControlFlowNode for Compare | 3 | diff --git a/python/ql/test/library-tests/PointsTo/version/VersionTest.ql b/python/ql/test/library-tests/PointsTo/version/VersionTest.ql deleted file mode 100644 index 0e6ca7fdee29..000000000000 --- a/python/ql/test/library-tests/PointsTo/version/VersionTest.ql +++ /dev/null @@ -1,9 +0,0 @@ - -import python -import semmle.python.types.Version - -from VersionTest vt, Location l, int v -where l = vt.getNode().getLocation() and -l.getFile().getName().matches("%test.py") -and (if vt.isTrue() then v = major_version() else v = 5-major_version()) -select l.getStartLine(), vt.(ControlFlowNode).toString(), v diff --git a/python/ql/test/library-tests/PointsTo/version/module.py b/python/ql/test/library-tests/PointsTo/version/module.py deleted file mode 100644 index 9e813a384287..000000000000 --- a/python/ql/test/library-tests/PointsTo/version/module.py +++ /dev/null @@ -1,5 +0,0 @@ - -import sys - -os_test = sys.platform == "linux" -version_test = sys.version_info < (3,) diff --git a/python/ql/test/library-tests/PointsTo/version/test.py b/python/ql/test/library-tests/PointsTo/version/test.py deleted file mode 100644 index a1c3783ba1bf..000000000000 --- a/python/ql/test/library-tests/PointsTo/version/test.py +++ /dev/null @@ -1,70 +0,0 @@ -import sys - - - - - - - - - - - - -os_test = sys.platform == "win32" -version_test = sys.version_info < (3,) - -from module import os_test as t2 -from module import version_test as t3 - - -# Tests from six -PY2 = sys.version_info[0] == 2 -PY3 = sys.version_info[0] == 3 - -if PY2: - version = 2 - -if PY3: - version = 3 - -if version == 2: - print("Version 2") - -if t2: - class G: pass -else: - def G(): pass - -g = G - -if t3: - class H: pass -else: - def H(): pass - -h = H - -#Some other forms of check. - -#Hexversion check (unlikely but a valid test) -PY2a = sys.hexversion < 0x03000000 -PY3a = sys.hexversion >= 0x03000000 - -PY2b = sys.hexversion < 0x03000000 -PY3b = sys.hexversion >= 0x03000000 - -PY2c = sys.version_info < (3,) -PY3c = sys.version_info >= (3,) -Py3d = sys.version_info >= (3,4) # Specific version of Python 3, rules out Python 2 -Py2d = sys.version_info < (2,7) -Py3e = sys.version_info[:2] >= (3,3) -Py2f = sys.version_info[:2] < (2,7) - -#From problem_report -Py2g = sys.version[0] < '3' -Py3h = sys.version[0] >= '3' - -if os_test: - pass - From 992ee2f15051092f3c94db24320ec49295a64d2e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 10 May 2019 16:03:27 +0100 Subject: [PATCH 081/128] Python points-to: Fix up some test results. --- .../PointsTo/new/PointsToUnknown.expected | 3 + .../library-tests/PointsTo/new/SSA.expected | 210 ++++-------------- .../PointsTo/new/Values.expected | 19 +- .../test/library-tests/PointsTo/new/Values.ql | 2 +- 4 files changed, 62 insertions(+), 172 deletions(-) diff --git a/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.expected b/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.expected index c715512184f9..93f0b8c515c7 100644 --- a/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.expected +++ b/python/ql/test/library-tests/PointsTo/new/PointsToUnknown.expected @@ -130,6 +130,7 @@ | c_tests.py:32 | ControlFlowNode for unknown | 32 | | c_tests.py:32 | ControlFlowNode for unknown() | 32 | | c_tests.py:32 | ControlFlowNode for y | 4 | +| c_tests.py:34 | ControlFlowNode for Attribute | 32 | | c_tests.py:34 | ControlFlowNode for Attribute | 34 | | c_tests.py:34 | ControlFlowNode for y | 4 | | c_tests.py:37 | ControlFlowNode for cond | 37 | @@ -198,7 +199,9 @@ | h_classes.py:17 | ControlFlowNode for arg | 14 | | h_classes.py:18 | ControlFlowNode for name | 18 | | h_classes.py:26 | ControlFlowNode for choice | 25 | +| h_classes.py:26 | ControlFlowNode for choice | 42 | | h_classes.py:28 | ControlFlowNode for choice | 25 | +| h_classes.py:28 | ControlFlowNode for choice | 42 | | h_classes.py:42 | ControlFlowNode for unknown | 42 | | h_classes.py:42 | ControlFlowNode for unknown() | 42 | | r_regressions.py:9 | ControlFlowNode for Attribute | 9 | diff --git a/python/ql/test/library-tests/PointsTo/new/SSA.expected b/python/ql/test/library-tests/PointsTo/new/SSA.expected index b0b215160922..5d8c600a4331 100644 --- a/python/ql/test/library-tests/PointsTo/new/SSA.expected +++ b/python/ql/test/library-tests/PointsTo/new/SSA.expected @@ -1,15 +1,11 @@ +WARNING: Predicate ssa_variable_points_to has been deprecated and may be removed in future (SSA.ql:10,1-33) | __init__.py:0 | __name___0 = ScopeEntryDefinition | 'code' | builtin-class str | | __init__.py:0 | __name___0 = ScopeEntryDefinition | 'code.package' | builtin-class str | | __init__.py:0 | __name___0 = ScopeEntryDefinition | 'code.test_package' | builtin-class str | -| __init__.py:0 | __package___0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | __init__.py:0 | module2_0 = ImplicitSubModuleDefinition | Module code.package.module2 | builtin-class module | | __init__.py:0 | moduleX_0 = ImplicitSubModuleDefinition | Module code.package.moduleX | builtin-class module | -| __init__.py:0 | sys_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | __init__.py:1 | __name___1 = ImportStarRefinement(__name___0) | 'code.test_package' | builtin-class str | -| __init__.py:1 | __package___1 = ImportStarRefinement(__package___0) | *UNDEFINED* | *UNKNOWN TYPE* | -| __init__.py:1 | sys_1 = ImportStarRefinement(sys_0) | *UNDEFINED* | *UNKNOWN TYPE* | | __init__.py:2 | __name___2 = ImportStarRefinement(__name___1) | 'code.test_package' | builtin-class str | -| __init__.py:2 | __package___2 = ImportStarRefinement(__package___1) | *UNDEFINED* | *UNKNOWN TYPE* | | __init__.py:2 | module_0 = ImportMember | Function module | builtin-class function | | __init__.py:3 | sys_2 = ImportExpr | Module sys | builtin-class module | | __init__.py:4 | module3_0 = ImportMember | Module code.package.module2 | builtin-class module | @@ -18,40 +14,19 @@ | __init__.py:7 | module5_0 = ImportMember | Module code.package.module2 | builtin-class module | | __init__.py:8 | moduleX_1 = ImportMember | Module code.package.moduleX | builtin-class module | | a_simple.py:0 | __name___0 = ScopeEntryDefinition | 'code.a_simple' | builtin-class str | -| a_simple.py:0 | __package___0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | a_simple.py:2 | f1_0 = FloatLiteral | float 1.0 | builtin-class float | | a_simple.py:5 | i1_0 = IntegerLiteral | int 0 | builtin-class int | | a_simple.py:6 | s_0 = Tuple | Tuple | builtin-class tuple | | a_simple.py:8 | func_0 = FunctionExpr | Function func | builtin-class function | | a_simple.py:11 | C_0 = ClassExpr | class C | builtin-class type | -| a_simple.py:14 | d_0 = ParameterDefinition | d | builtin-class dict | -| a_simple.py:14 | t_0 = ParameterDefinition | t | builtin-class tuple | | a_simple.py:14 | vararg_kwarg_0 = FunctionExpr | Function vararg_kwarg | builtin-class function | | a_simple.py:18 | multi_loop_0 = FunctionExpr | Function multi_loop | builtin-class function | -| a_simple.py:18 | y_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | a_simple.py:19 | x_0 = None | NoneType None | builtin-class NoneType | | a_simple.py:20 | x_1 = phi(x_0, x_2) | NoneType None | builtin-class NoneType | -| a_simple.py:20 | y_1 = phi(y_0, y_2) | *UNDEFINED* | *UNKNOWN TYPE* | | a_simple.py:23 | with_definition_0 = FunctionExpr | Function with_definition | builtin-class function | | a_simple.py:27 | multi_loop_in_try_0 = FunctionExpr | Function multi_loop_in_try | builtin-class function | -| a_simple.py:27 | p_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| a_simple.py:27 | q_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| a_simple.py:29 | p_1 = phi(p_0, p_2) | *UNDEFINED* | *UNKNOWN TYPE* | -| a_simple.py:29 | q_1 = phi(q_0, q_2) | *UNDEFINED* | *UNKNOWN TYPE* | -| a_simple.py:34 | args_0 = ParameterDefinition | args | builtin-class tuple | | a_simple.py:34 | f_0 = FunctionExpr | Function f | builtin-class function | -| a_simple.py:34 | kwargs_0 = ParameterDefinition | kwargs | builtin-class dict | | b_condition.py:0 | __name___0 = ScopeEntryDefinition | 'code.b_condition' | builtin-class str | -| b_condition.py:0 | __package___0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| b_condition.py:0 | double_attr_check_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| b_condition.py:0 | g_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| b_condition.py:0 | h_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| b_condition.py:0 | k_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| b_condition.py:0 | loop_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| b_condition.py:0 | not_or_not_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| b_condition.py:0 | odasa6261_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| b_condition.py:0 | split_bool1_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| b_condition.py:0 | v2_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | b_condition.py:4 | f_0 = FunctionExpr | Function f | builtin-class function | | b_condition.py:5 | x_0 = IfExp | NoneType None | builtin-class NoneType | | b_condition.py:8 | x_1 = IntegerLiteral | int 7 | builtin-class int | @@ -82,8 +57,6 @@ | b_condition.py:37 | x_24 = ArgumentRefinement(x_23) | int 7 | builtin-class int | | b_condition.py:50 | g_1 = FunctionExpr | Function g | builtin-class function | | b_condition.py:55 | loop_1 = FunctionExpr | Function loop | builtin-class function | -| b_condition.py:55 | v_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| b_condition.py:56 | v_2 = phi(v_0, v_1, v_5) | *UNDEFINED* | *UNKNOWN TYPE* | | b_condition.py:61 | double_attr_check_1 = FunctionExpr | Function double_attr_check | builtin-class function | | b_condition.py:69 | h_1 = FunctionExpr | Function h | builtin-class function | | b_condition.py:70 | b_0 = IfExp | bool True | builtin-class bool | @@ -96,15 +69,12 @@ | b_condition.py:78 | t_1 = object | builtin-class object | builtin-class type | | b_condition.py:79 | t_3 = phi(t_1, t_2) | builtin-class object | builtin-class type | | b_condition.py:79 | t_4 = ArgumentRefinement(t_3) | builtin-class object | builtin-class type | -| b_condition.py:81 | bar_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| b_condition.py:81 | bar_2 = phi(bar_0, bar_1) | *UNDEFINED* | *UNKNOWN TYPE* | | b_condition.py:81 | bar_2 = phi(bar_0, bar_1) | Function bar | builtin-class function | | b_condition.py:81 | foo_0 = ParameterDefinition | bool True | builtin-class bool | | b_condition.py:81 | foo_3 = Pi(foo_0) [false] | bool True | builtin-class bool | | b_condition.py:81 | foo_4 = phi(foo_1, foo_3) | bool True | builtin-class bool | | b_condition.py:81 | odasa6261_1 = FunctionExpr | Function odasa6261 | builtin-class function | | b_condition.py:83 | bar_1 = FunctionExpr | Function bar | builtin-class function | -| b_condition.py:83 | foo_2 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | b_condition.py:87 | split_bool1_1 = FunctionExpr | Function split_bool1 | builtin-class function | | b_condition.py:87 | x_0 = ParameterDefinition | NoneType None | builtin-class NoneType | | b_condition.py:87 | y_0 = ParameterDefinition | NoneType None | builtin-class NoneType | @@ -115,47 +85,48 @@ | b_condition.py:93 | y_5 = ArgumentRefinement(y_4) | NoneType None | builtin-class NoneType | | b_condition.py:96 | y_6 = SingleSuccessorGuard(y_5) [false] | NoneType None | builtin-class NoneType | | b_condition.py:97 | x_3 = ArgumentRefinement(x_2) | NoneType None | builtin-class NoneType | -| b_condition.py:101 | a_0 = ParameterDefinition | a | builtin-class tuple | | b_condition.py:101 | not_or_not_1 = FunctionExpr | Function not_or_not | builtin-class function | -| b_condition.py:104 | a_1 = Pi(a_0) [false] | a | builtin-class tuple | -| b_condition.py:105 | a_2 = Pi(a_1) [false] | a | builtin-class tuple | -| b_condition.py:107 | a_3 = Pi(a_2) [false] | a | builtin-class tuple | | c_tests.py:0 | __name___0 = ScopeEntryDefinition | 'code.c_tests' | builtin-class str | -| c_tests.py:0 | __package___0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | c_tests.py:4 | f_0 = FunctionExpr | Function f | builtin-class function | | c_tests.py:5 | x_0 = IfExp | NoneType None | builtin-class NoneType | -| c_tests.py:10 | x_1 = IfExp | int 0 | builtin-class int | -| c_tests.py:10 | x_1 = IfExp | int 1 | builtin-class int | -| c_tests.py:15 | x_2 = IfExp | int 0 | builtin-class int | -| c_tests.py:15 | x_2 = IfExp | int 1 | builtin-class int | -| c_tests.py:21 | x_3 = IfExp | List | builtin-class list | -| c_tests.py:21 | x_3 = IfExp | Tuple | builtin-class tuple | -| c_tests.py:24 | x_4 = Pi(x_3) [true] | List | builtin-class list | -| c_tests.py:24 | x_4 = Pi(x_3) [true] | Tuple | builtin-class tuple | -| c_tests.py:26 | x_6 = phi(x_4, x_5) | List | builtin-class list | -| c_tests.py:26 | x_6 = phi(x_4, x_5) | Tuple | builtin-class tuple | -| c_tests.py:27 | x_7 = Pi(x_6) [true] | List | builtin-class list | -| c_tests.py:27 | x_7 = Pi(x_6) [true] | Tuple | builtin-class tuple | -| c_tests.py:29 | x_8 = Pi(x_6) [false] | Tuple | builtin-class tuple | -| c_tests.py:29 | x_9 = phi(x_7, x_8) | List | builtin-class list | -| c_tests.py:29 | x_9 = phi(x_7, x_8) | Tuple | builtin-class tuple | -| c_tests.py:30 | x_10 = Pi(x_9) [true] | Tuple | builtin-class tuple | -| c_tests.py:32 | x_11 = Pi(x_9) [false] | List | builtin-class list | -| c_tests.py:32 | x_12 = phi(x_10, x_11) | List | builtin-class list | -| c_tests.py:32 | x_12 = phi(x_10, x_11) | Tuple | builtin-class tuple | +| c_tests.py:8 | x_1 = Pi(x_0) [true] | NoneType None | builtin-class NoneType | +| c_tests.py:10 | x_2 = IfExp | int 0 | builtin-class int | +| c_tests.py:10 | x_2 = IfExp | int 1 | builtin-class int | +| c_tests.py:13 | x_3 = Pi(x_2) [true] | int 1 | builtin-class int | +| c_tests.py:15 | x_4 = IfExp | int 0 | builtin-class int | +| c_tests.py:15 | x_4 = IfExp | int 1 | builtin-class int | +| c_tests.py:18 | x_5 = Pi(x_4) [true] | int 0 | builtin-class int | +| c_tests.py:21 | x_6 = IfExp | List | builtin-class list | +| c_tests.py:21 | x_6 = IfExp | Tuple | builtin-class tuple | +| c_tests.py:24 | x_7 = Pi(x_6) [true] | List | builtin-class list | +| c_tests.py:24 | x_7 = Pi(x_6) [true] | Tuple | builtin-class tuple | +| c_tests.py:26 | x_8 = Pi(x_6) [false] | List | builtin-class list | +| c_tests.py:26 | x_9 = phi(x_7, x_8) | List | builtin-class list | +| c_tests.py:26 | x_9 = phi(x_7, x_8) | Tuple | builtin-class tuple | +| c_tests.py:27 | x_10 = Pi(x_9) [true] | List | builtin-class list | +| c_tests.py:27 | x_10 = Pi(x_9) [true] | Tuple | builtin-class tuple | +| c_tests.py:29 | x_11 = Pi(x_9) [false] | List | builtin-class list | +| c_tests.py:29 | x_11 = Pi(x_9) [false] | Tuple | builtin-class tuple | +| c_tests.py:29 | x_12 = phi(x_10, x_11) | List | builtin-class list | +| c_tests.py:29 | x_12 = phi(x_10, x_11) | Tuple | builtin-class tuple | +| c_tests.py:30 | x_13 = Pi(x_12) [true] | Tuple | builtin-class tuple | +| c_tests.py:32 | x_14 = Pi(x_12) [false] | List | builtin-class list | +| c_tests.py:32 | x_15 = phi(x_13, x_14) | List | builtin-class list | +| c_tests.py:32 | x_15 = phi(x_13, x_14) | Tuple | builtin-class tuple | | c_tests.py:56 | others_0 = FunctionExpr | Function others | builtin-class function | -| c_tests.py:56 | x_6 = Pi(x_4) [false] | int 0 | builtin-class int | -| c_tests.py:56 | x_7 = phi(x_5, x_6) | builtin-class float | builtin-class type | -| c_tests.py:56 | x_7 = phi(x_5, x_6) | int 0 | builtin-class int | +| c_tests.py:56 | x_7 = Pi(x_5) [false] | int 0 | builtin-class int | +| c_tests.py:56 | x_8 = phi(x_6, x_7) | builtin-class float | builtin-class type | +| c_tests.py:56 | x_8 = phi(x_6, x_7) | int 0 | builtin-class int | | c_tests.py:58 | x_0 = IfExp | builtin-class bool | builtin-class type | | c_tests.py:58 | x_0 = IfExp | builtin-class type | builtin-class type | -| c_tests.py:63 | x_1 = IfExp | builtin-class float | builtin-class type | -| c_tests.py:63 | x_1 = IfExp | int 0 | builtin-class int | -| c_tests.py:66 | x_2 = Pi(x_1) [true] | int 0 | builtin-class int | -| c_tests.py:68 | x_3 = Pi(x_1) [false] | builtin-class float | builtin-class type | -| c_tests.py:68 | x_4 = phi(x_2, x_3) | builtin-class float | builtin-class type | -| c_tests.py:68 | x_4 = phi(x_2, x_3) | int 0 | builtin-class int | -| c_tests.py:69 | x_5 = Pi(x_4) [true] | builtin-class float | builtin-class type | +| c_tests.py:61 | x_1 = Pi(x_0) [true] | builtin-class bool | builtin-class type | +| c_tests.py:63 | x_2 = IfExp | builtin-class float | builtin-class type | +| c_tests.py:63 | x_2 = IfExp | int 0 | builtin-class int | +| c_tests.py:66 | x_3 = Pi(x_2) [true] | int 0 | builtin-class int | +| c_tests.py:68 | x_4 = Pi(x_2) [false] | builtin-class float | builtin-class type | +| c_tests.py:68 | x_5 = phi(x_3, x_4) | builtin-class float | builtin-class type | +| c_tests.py:68 | x_5 = phi(x_3, x_4) | int 0 | builtin-class int | +| c_tests.py:69 | x_6 = Pi(x_5) [true] | builtin-class float | builtin-class type | | c_tests.py:71 | compound_0 = FunctionExpr | Function compound | builtin-class function | | c_tests.py:71 | x_0 = ParameterDefinition | int 1 | builtin-class int | | c_tests.py:71 | y_0 = ParameterDefinition | int 0 | builtin-class int | @@ -166,45 +137,24 @@ | c_tests.py:76 | y_2 = Pi(y_0) [false] | int 0 | builtin-class int | | c_tests.py:76 | y_3 = phi(y_1, y_2) | int 0 | builtin-class int | | c_tests.py:79 | h_0 = FunctionExpr | Function h | builtin-class function | -| c_tests.py:79 | x_4 = phi(x_2, x_3) | NoneType None | builtin-class NoneType | +| c_tests.py:79 | x_5 = phi(x_3, x_4) | NoneType None | builtin-class NoneType | | c_tests.py:80 | b_0 = IfExp | bool True | builtin-class bool | -| c_tests.py:83 | b_1 = IfExp | bool True | builtin-class bool | -| c_tests.py:87 | b_3 = Pi(b_1) [false] | bool True | builtin-class bool | -| c_tests.py:87 | b_4 = phi(b_2, b_3) | bool True | builtin-class bool | +| c_tests.py:82 | b_1 = Pi(b_0) [true] | bool True | builtin-class bool | +| c_tests.py:83 | b_2 = IfExp | bool True | builtin-class bool | +| c_tests.py:87 | b_4 = Pi(b_2) [false] | bool True | builtin-class bool | +| c_tests.py:87 | b_5 = phi(b_3, b_4) | bool True | builtin-class bool | | c_tests.py:90 | x_0 = IfExp | NoneType None | builtin-class NoneType | -| c_tests.py:94 | x_1 = IfExp | NoneType None | builtin-class NoneType | -| c_tests.py:96 | x_2 = Pi(x_1) [true] | NoneType None | builtin-class NoneType | +| c_tests.py:94 | x_2 = IfExp | NoneType None | builtin-class NoneType | +| c_tests.py:96 | x_3 = Pi(x_2) [true] | NoneType None | builtin-class NoneType | | c_tests.py:98 | complex_test_0 = FunctionExpr | Function complex_test | builtin-class function | -| d_globals.py:0 | D_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | Ugly_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | X_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:0 | __name___0 = ScopeEntryDefinition | 'code.d_globals' | builtin-class str | -| d_globals.py:0 | __package___0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | dict_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | g3_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | g4_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | get_g4_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | glob_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | k_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | modinit_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | outer_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | redefine_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | set_g4_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | set_g4_indirect_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | tuple_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | use_list_attribute_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | x_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | y_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:0 | z_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:2 | dict_2 = ScopeEntryDefinition | int 7 | builtin-class int | | d_globals.py:2 | g1_2 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:2 | g2_2 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:2 | g3_2 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:2 | g4_1 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:2 | glob_2 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:2 | j_0 = FunctionExpr | Function j | builtin-class function | | d_globals.py:2 | tuple_2 = ScopeEntryDefinition | builtin-class tuple | builtin-class type | -| d_globals.py:2 | z_2 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:5 | dict_1 = IntegerLiteral | int 7 | builtin-class int | | d_globals.py:7 | tuple_1 = tuple | builtin-class tuple | builtin-class type | | d_globals.py:14 | g1_0 = None | NoneType None | builtin-class NoneType | @@ -212,41 +162,29 @@ | d_globals.py:16 | g2_3 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:16 | g3_3 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:16 | g4_2 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:16 | glob_3 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:16 | z_3 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:18 | g1_3 = IntegerLiteral | int 101 | builtin-class int | | d_globals.py:23 | g2_0 = None | NoneType None | builtin-class NoneType | | d_globals.py:25 | g1_4 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:25 | g3_4 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:25 | g3_4 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:25 | g4_3 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:25 | g4_3 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:25 | glob_4 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:25 | init_0 = FunctionExpr | Function init | builtin-class function | -| d_globals.py:25 | z_4 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:27 | g2_4 = IntegerLiteral | int 102 | builtin-class int | | d_globals.py:29 | g1_1 = CallsiteRefinement(g1_0) | NoneType None | builtin-class NoneType | | d_globals.py:29 | g2_1 = CallsiteRefinement(g2_0) | int 102 | builtin-class int | -| d_globals.py:29 | glob_1 = CallsiteRefinement(glob_0) | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:29 | z_1 = CallsiteRefinement(z_0) | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:33 | g3_1 = None | NoneType None | builtin-class NoneType | | d_globals.py:35 | Ugly_1 = ClassExpr | class Ugly | builtin-class type | | d_globals.py:37 | __init___0 = FunctionExpr | Function __init__ | builtin-class function | | d_globals.py:37 | g1_5 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:37 | g2_5 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:37 | g4_4 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:37 | glob_5 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:37 | self_0 = ParameterDefinition | self | class Ugly | -| d_globals.py:37 | z_5 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:39 | g3_5 = IntegerLiteral | int 103 | builtin-class int | | d_globals.py:41 | g1_6 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:41 | g2_6 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:41 | g3_6 = ScopeEntryDefinition | int 103 | builtin-class int | | d_globals.py:41 | g4_5 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:41 | glob_6 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:41 | meth_0 = FunctionExpr | Function meth | builtin-class function | | d_globals.py:41 | self_0 = ParameterDefinition | self | class Ugly | -| d_globals.py:41 | z_6 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:46 | x_1 = IntegerLiteral | int 1 | builtin-class int | | d_globals.py:49 | x_2 = IntegerLiteral | int 3 | builtin-class int | | d_globals.py:51 | x_3 = phi(x_1, x_2) | int 1 | builtin-class int | @@ -256,9 +194,7 @@ | d_globals.py:59 | y_3 = phi(y_1, y_2) | int 1 | builtin-class int | | d_globals.py:59 | y_3 = phi(y_1, y_2) | int 2 | builtin-class int | | d_globals.py:62 | X_1 = ClassExpr | class X | builtin-class type | -| d_globals.py:62 | X_2 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:62 | g3_7 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:62 | y_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:62 | y_4 = ScopeEntryDefinition | int 1 | builtin-class int | | d_globals.py:62 | y_4 = ScopeEntryDefinition | int 2 | builtin-class int | | d_globals.py:63 | y_1 = y | int 1 | builtin-class int | @@ -267,88 +203,65 @@ | d_globals.py:70 | g2_7 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:70 | g3_8 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:70 | g4_7 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:70 | glob_7 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:70 | k_1 = FunctionExpr | Function k | builtin-class function | -| d_globals.py:70 | z_7 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:73 | g4_6 = None | NoneType None | builtin-class NoneType | | d_globals.py:75 | g1_8 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:75 | g2_8 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:75 | g3_9 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:75 | g4_8 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:75 | get_g4_1 = FunctionExpr | Function get_g4 | builtin-class function | -| d_globals.py:75 | glob_8 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:75 | set_g4_2 = ScopeEntryDefinition | Function set_g4 | builtin-class function | -| d_globals.py:75 | z_8 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:77 | g1_9 = CallsiteRefinement(g1_8) | NoneType None | builtin-class NoneType | | d_globals.py:77 | g2_9 = CallsiteRefinement(g2_8) | int 102 | builtin-class int | | d_globals.py:77 | g3_10 = CallsiteRefinement(g3_9) | NoneType None | builtin-class NoneType | | d_globals.py:77 | g4_9 = Pi(g4_8) [true] | NoneType None | builtin-class NoneType | | d_globals.py:77 | g4_10 = CallsiteRefinement(g4_9) | bool False | builtin-class bool | -| d_globals.py:77 | glob_9 = CallsiteRefinement(glob_8) | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:77 | z_9 = CallsiteRefinement(z_8) | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:78 | g1_10 = phi(g1_8, g1_9) | NoneType None | builtin-class NoneType | | d_globals.py:78 | g2_10 = phi(g2_8, g2_9) | int 102 | builtin-class int | | d_globals.py:78 | g3_11 = phi(g3_9, g3_10) | NoneType None | builtin-class NoneType | | d_globals.py:78 | g4_12 = phi(g4_10, g4_11) | bool False | builtin-class bool | -| d_globals.py:78 | glob_10 = phi(glob_8, glob_9) | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:78 | z_10 = phi(z_8, z_9) | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:80 | g1_11 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:80 | g2_11 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:80 | g3_12 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:80 | g4_13 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:80 | glob_11 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:80 | set_g4_1 = FunctionExpr | Function set_g4 | builtin-class function | | d_globals.py:80 | set_g4_indirect_2 = ScopeEntryDefinition | Function set_g4_indirect | builtin-class function | -| d_globals.py:80 | z_11 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:81 | g1_12 = CallsiteRefinement(g1_11) | NoneType None | builtin-class NoneType | | d_globals.py:81 | g2_12 = CallsiteRefinement(g2_11) | int 102 | builtin-class int | | d_globals.py:81 | g3_13 = CallsiteRefinement(g3_12) | NoneType None | builtin-class NoneType | | d_globals.py:81 | g4_14 = CallsiteRefinement(g4_13) | bool False | builtin-class bool | -| d_globals.py:81 | glob_12 = CallsiteRefinement(glob_11) | *UNDEFINED* | *UNKNOWN TYPE* | -| d_globals.py:81 | z_12 = CallsiteRefinement(z_11) | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:83 | g1_13 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:83 | g2_13 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:83 | g3_14 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:83 | glob_13 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:83 | set_g4_indirect_1 = FunctionExpr | Function set_g4_indirect | builtin-class function | -| d_globals.py:83 | z_13 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:85 | g4_15 = False | bool False | builtin-class bool | | d_globals.py:87 | modinit_1 = ClassExpr | class modinit | builtin-class type | -| d_globals.py:92 | modinit_2 = DeletionDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:95 | g1_14 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:95 | g2_14 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:95 | g3_15 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:95 | g4_16 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:95 | glob_14 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:95 | outer_1 = FunctionExpr | Function outer | builtin-class function | -| d_globals.py:95 | z_14 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:96 | g1_16 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:96 | g2_16 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:96 | g3_17 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:96 | g4_18 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:96 | inner_0 = FunctionExpr | Function inner | builtin-class function | -| d_globals.py:96 | z_16 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:98 | glob_16 = IntegerLiteral | int 100 | builtin-class int | | d_globals.py:101 | g1_17 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:101 | g2_17 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:101 | g3_18 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:101 | g4_19 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:101 | glob_17 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:101 | otherInner_0 = FunctionExpr | Function otherInner | builtin-class function | -| d_globals.py:101 | z_17 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:104 | g1_15 = CallsiteRefinement(g1_14) | NoneType None | builtin-class NoneType | | d_globals.py:104 | g2_15 = CallsiteRefinement(g2_14) | int 102 | builtin-class int | | d_globals.py:104 | g3_16 = CallsiteRefinement(g3_15) | NoneType None | builtin-class NoneType | | d_globals.py:104 | g4_17 = CallsiteRefinement(g4_16) | NoneType None | builtin-class NoneType | | d_globals.py:104 | glob_15 = CallsiteRefinement(glob_14) | int 100 | builtin-class int | -| d_globals.py:104 | z_15 = CallsiteRefinement(z_14) | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:107 | g1_18 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:107 | g2_18 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:107 | g3_19 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:107 | g4_20 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:107 | glob_18 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:107 | redefine_1 = FunctionExpr | Function redefine | builtin-class function | -| d_globals.py:107 | z_18 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:110 | z_19 = IntegerLiteral | int 1 | builtin-class int | | d_globals.py:113 | glob_19 = IntegerLiteral | int 50 | builtin-class int | | d_globals.py:118 | D_1 = ClassExpr | class D | builtin-class type | @@ -357,36 +270,26 @@ | d_globals.py:120 | g2_19 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:120 | g3_20 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:120 | g4_21 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:120 | glob_20 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:120 | self_0 = ParameterDefinition | self | class D | -| d_globals.py:120 | z_20 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:123 | dict_3 = ScopeEntryDefinition | int 7 | builtin-class int | | d_globals.py:123 | foo_0 = FunctionExpr | Function foo | builtin-class function | | d_globals.py:123 | g1_20 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:123 | g2_20 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:123 | g3_21 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:123 | g4_22 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:123 | glob_21 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:123 | self_0 = ParameterDefinition | self | class D | -| d_globals.py:123 | z_21 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:126 | g1_21 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:126 | g2_21 = ScopeEntryDefinition | int 102 | builtin-class int | | d_globals.py:126 | g3_22 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | | d_globals.py:126 | g4_23 = ScopeEntryDefinition | NoneType None | builtin-class NoneType | -| d_globals.py:126 | glob_22 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:126 | use_list_attribute_1 = FunctionExpr | Function use_list_attribute | builtin-class function | -| d_globals.py:126 | z_22 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:127 | l_0 = List | List | builtin-class list | | d_globals.py:128 | g1_22 = CallsiteRefinement(g1_21) | NoneType None | builtin-class NoneType | | d_globals.py:128 | g2_22 = CallsiteRefinement(g2_21) | int 102 | builtin-class int | | d_globals.py:128 | g3_23 = CallsiteRefinement(g3_22) | NoneType None | builtin-class NoneType | | d_globals.py:128 | g4_24 = CallsiteRefinement(g4_23) | NoneType None | builtin-class NoneType | -| d_globals.py:128 | glob_23 = CallsiteRefinement(glob_22) | *UNDEFINED* | *UNKNOWN TYPE* | | d_globals.py:128 | l_1 = ArgumentRefinement(l_0) | List | builtin-class list | -| d_globals.py:128 | z_23 = CallsiteRefinement(z_22) | *UNDEFINED* | *UNKNOWN TYPE* | | e_temporal.py:0 | __name___0 = ScopeEntryDefinition | 'code.e_temporal' | builtin-class str | -| e_temporal.py:0 | __package___0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| e_temporal.py:0 | x_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | e_temporal.py:2 | sys_0 = ImportExpr | Module sys | builtin-class module | | e_temporal.py:4 | f_0 = FunctionExpr | Function f | builtin-class function | | e_temporal.py:4 | sys_1 = ScopeEntryDefinition | Module sys | builtin-class module | @@ -394,7 +297,6 @@ | e_temporal.py:9 | g_0 = FunctionExpr | Function g | builtin-class function | | e_temporal.py:12 | x_1 = g() | int 1 | builtin-class int | | g_class_init.py:0 | __name___0 = ScopeEntryDefinition | 'code.g_class_init' | builtin-class str | -| g_class_init.py:0 | __package___0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | g_class_init.py:3 | C_0 = ClassExpr | class C | builtin-class type | | g_class_init.py:5 | __init___0 = FunctionExpr | Function __init__ | builtin-class function | | g_class_init.py:5 | self_0 = ParameterDefinition | self | class C | @@ -413,8 +315,6 @@ | g_class_init.py:20 | self_2 = Pi(self_0) [false] | self | class C | | g_class_init.py:20 | self_3 = phi(self_1, self_2) | self | class C | | g_class_init.py:24 | Oddities_0 = ClassExpr | class Oddities | builtin-class type | -| g_class_init.py:24 | float_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| g_class_init.py:24 | int_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | g_class_init.py:26 | int_1 = int | builtin-class int | builtin-class type | | g_class_init.py:27 | float_1 = float | builtin-class float | builtin-class type | | g_class_init.py:28 | l_0 = len | Builtin-function len | builtin-class builtin_function_or_method | @@ -441,7 +341,6 @@ | g_class_init.py:52 | self_3 = phi(self_1, self_2) | self | class E | | g_class_init.py:54 | self_1 = Pi(self_0) [true] | self | class E | | j_convoluted_imports.py:0 | __name___0 = ScopeEntryDefinition | 'code.j_convoluted_imports' | builtin-class str | -| j_convoluted_imports.py:0 | __package___0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | j_convoluted_imports.py:3 | module_0 = ImportMember | Function module | builtin-class function | | j_convoluted_imports.py:6 | x_0 = ImportMember | Module code.package.x | builtin-class module | | j_convoluted_imports.py:9 | C_0 = ClassExpr | class C | builtin-class type | @@ -451,7 +350,6 @@ | j_convoluted_imports.py:14 | x_0 = ImportMember | Module code.package.x | builtin-class module | | j_convoluted_imports.py:16 | moduleX_0 = ImportMember | Module code.package.moduleX | builtin-class module | | m_attributes.py:0 | __name___0 = ScopeEntryDefinition | 'code.m_attributes' | builtin-class str | -| m_attributes.py:0 | __package___0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | m_attributes.py:3 | C_0 = ClassExpr | class C | builtin-class type | | m_attributes.py:5 | __init___0 = FunctionExpr | Function __init__ | builtin-class function | | m_attributes.py:5 | a_0 = ParameterDefinition | int 17 | builtin-class int | @@ -460,21 +358,18 @@ | m_attributes.py:6 | self_1 = AttributeAssignment 'a'(self_0) | self | class C | | m_attributes.py:8 | foo_0 = FunctionExpr | Function foo | builtin-class function | | m_attributes.py:8 | other_0 = ParameterDefinition | C() | class C | +| m_attributes.py:8 | self_0 = ParameterDefinition | C() | class C | | m_attributes.py:8 | self_0 = ParameterDefinition | self | class C | -| n_nesting.py:0 | D_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | n_nesting.py:0 | __name___0 = ScopeEntryDefinition | 'code.n_nesting' | builtin-class str | -| n_nesting.py:0 | __package___0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | n_nesting.py:8 | C_0 = ScopeEntryDefinition | int 1 | builtin-class int | | n_nesting.py:8 | compile_ops_0 = ParameterDefinition | bool True | builtin-class bool | | n_nesting.py:8 | foo_0 = FunctionExpr | Function foo | builtin-class function | | n_nesting.py:9 | C_1 = CallsiteRefinement(C_0) | int 1 | builtin-class int | | n_nesting.py:10 | C_5 = ScopeEntryDefinition | int 1 | builtin-class int | -| n_nesting.py:10 | compile_ops_2 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | n_nesting.py:10 | inner_0 = FunctionExpr | Function inner | builtin-class function | | n_nesting.py:11 | C_6 = CallsiteRefinement(C_5) | int 1 | builtin-class int | | n_nesting.py:13 | C_7 = ScopeEntryDefinition | int 1 | builtin-class int | | n_nesting.py:13 | compile_ops_3 = Pi(compile_ops_0) [false] | bool True | builtin-class bool | -| n_nesting.py:13 | compile_ops_4 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | n_nesting.py:13 | inner_1 = FunctionExpr | Function inner | builtin-class function | | n_nesting.py:14 | C_8 = CallsiteRefinement(C_7) | int 1 | builtin-class int | | n_nesting.py:15 | attrs_0 = Dict | Dict | builtin-class dict | @@ -506,7 +401,6 @@ | n_nesting.py:32 | D_1 = ClassExpr | class D | builtin-class type | | n_nesting.py:34 | C_4 = IntegerLiteral | int 1 | builtin-class int | | q_super.py:0 | __name___0 = ScopeEntryDefinition | 'code.q_super' | builtin-class str | -| q_super.py:0 | __package___0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | q_super.py:1 | Base2_0 = ClassExpr | class Base2 | builtin-class type | | q_super.py:3 | Base2_1 = ScopeEntryDefinition | class Base2 | builtin-class type | | q_super.py:3 | __init___0 = FunctionExpr | Function __init__ | builtin-class function | @@ -556,14 +450,10 @@ | q_super.py:50 | DB_1 = ScopeEntryDefinition | class DB | builtin-class type | | q_super.py:50 | __init___0 = FunctionExpr | Function __init__ | builtin-class function | | q_super.py:50 | self_0 = ParameterDefinition | self | class DC | -| q_super.py:51 | sup_0 = super() | super() | builtin-class super | -| q_super.py:52 | sup_1 = MethodCallsiteRefinement(sup_0) | super() | builtin-class super | | q_super.py:55 | DD_0 = ClassExpr | class DD | builtin-class type | | q_super.py:57 | DD_1 = ScopeEntryDefinition | class DD | builtin-class type | | q_super.py:57 | __init___0 = FunctionExpr | Function __init__ | builtin-class function | | q_super.py:57 | self_0 = ParameterDefinition | self | class DD | -| q_super.py:58 | sup_0 = super() | super() | builtin-class super | -| q_super.py:59 | sup_1 = MethodCallsiteRefinement(sup_0) | super() | builtin-class super | | q_super.py:61 | DA_2 = ScopeEntryDefinition | class DA | builtin-class type | | q_super.py:61 | DE_0 = ClassExpr | class DE | builtin-class type | | q_super.py:63 | DF_0 = ClassExpr | class DF | builtin-class type | @@ -575,31 +465,19 @@ | q_super.py:73 | M_1 = ScopeEntryDefinition | class M | builtin-class type | | q_super.py:73 | __init___0 = FunctionExpr | Function __init__ | builtin-class function | | q_super.py:73 | self_0 = ParameterDefinition | self | class M | -| q_super.py:74 | s_0 = super() | super() | builtin-class super | -| q_super.py:75 | i_0 = Attribute | super().__init__ | builtin-class method | | s_scopes.py:0 | __name___0 = ScopeEntryDefinition | 'code.s_scopes' | builtin-class str | -| s_scopes.py:0 | __package___0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| s_scopes.py:0 | float_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| s_scopes.py:0 | x_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | s_scopes.py:4 | float_1 = True | bool True | builtin-class bool | -| s_scopes.py:5 | float_2 = phi(float_0, float_1) | *UNDEFINED* | *UNKNOWN TYPE* | | s_scopes.py:5 | float_2 = phi(float_0, float_1) | bool True | builtin-class bool | | s_scopes.py:7 | C2_0 = ClassExpr | class C2 | builtin-class type | -| s_scopes.py:7 | float_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| s_scopes.py:7 | float_3 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | s_scopes.py:7 | float_3 = ScopeEntryDefinition | bool True | builtin-class bool | -| s_scopes.py:7 | int_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | -| s_scopes.py:7 | str_0 = ScopeEntryDefinition | *UNDEFINED* | *UNKNOWN TYPE* | | s_scopes.py:9 | i1_0 = int | builtin-class int | builtin-class type | | s_scopes.py:10 | f1_0 = float | bool True | builtin-class bool | | s_scopes.py:10 | f1_0 = float | builtin-class float | builtin-class type | | s_scopes.py:12 | int_1 = IntegerLiteral | int 0 | builtin-class int | | s_scopes.py:15 | str_1 = FloatLiteral | float 1.0 | builtin-class float | | s_scopes.py:17 | float_1 = None | NoneType None | builtin-class NoneType | -| s_scopes.py:18 | float_2 = phi(float_0, float_1) | *UNDEFINED* | *UNKNOWN TYPE* | | s_scopes.py:18 | float_2 = phi(float_0, float_1) | NoneType None | builtin-class NoneType | | s_scopes.py:18 | i2_0 = int | int 0 | builtin-class int | -| s_scopes.py:18 | str_2 = phi(str_0, str_1) | *UNDEFINED* | *UNKNOWN TYPE* | | s_scopes.py:18 | str_2 = phi(str_0, str_1) | float 1.0 | builtin-class float | | s_scopes.py:19 | s_0 = str | builtin-class str | builtin-class type | | s_scopes.py:19 | s_0 = str | float 1.0 | builtin-class float | diff --git a/python/ql/test/library-tests/PointsTo/new/Values.expected b/python/ql/test/library-tests/PointsTo/new/Values.expected index a388d1c2c897..0283e21fb5a8 100644 --- a/python/ql/test/library-tests/PointsTo/new/Values.expected +++ b/python/ql/test/library-tests/PointsTo/new/Values.expected @@ -123,10 +123,22 @@ | b_condition.py:97 | ControlFlowNode for x | runtime | None | builtin-class NoneType | | b_condition.py:101 | ControlFlowNode for FunctionExpr | import | Function not_or_not | builtin-class function | | b_condition.py:102 | ControlFlowNode for Tuple | runtime | (builtin-class tuple, builtin-class list) | builtin-class tuple | +| b_condition.py:102 | ControlFlowNode for UnaryExpr | runtime | bool False | builtin-class bool | | b_condition.py:102 | ControlFlowNode for a | runtime | instance of tuple | builtin-class tuple | | b_condition.py:102 | ControlFlowNode for isinstance | runtime | Builtin-function isinstance | builtin-class builtin_function_or_method | +| b_condition.py:102 | ControlFlowNode for isinstance() | runtime | bool True | builtin-class bool | | b_condition.py:102 | ControlFlowNode for list | runtime | builtin-class list | builtin-class type | | b_condition.py:102 | ControlFlowNode for tuple | runtime | builtin-class tuple | builtin-class type | +| b_condition.py:104 | ControlFlowNode for UnaryExpr | runtime | bool False | builtin-class bool | +| b_condition.py:104 | ControlFlowNode for UnaryExpr | runtime | bool True | builtin-class bool | +| b_condition.py:104 | ControlFlowNode for a | runtime | instance of tuple | builtin-class tuple | +| b_condition.py:105 | ControlFlowNode for IntegerLiteral | runtime | int 0 | builtin-class int | +| b_condition.py:105 | ControlFlowNode for UnaryExpr | runtime | bool False | builtin-class bool | +| b_condition.py:105 | ControlFlowNode for UnaryExpr | runtime | bool True | builtin-class bool | +| b_condition.py:105 | ControlFlowNode for a | runtime | instance of tuple | builtin-class tuple | +| b_condition.py:106 | ControlFlowNode for Exception | runtime | builtin-class Exception | builtin-class type | +| b_condition.py:106 | ControlFlowNode for Exception() | runtime | Exception() | builtin-class Exception | +| b_condition.py:107 | ControlFlowNode for Str | runtime | 'Hello' | builtin-class str | | e_temporal.py:2 | ControlFlowNode for ImportExpr | import | Module sys | builtin-class module | | e_temporal.py:4 | ControlFlowNode for FunctionExpr | import | Function f | builtin-class function | | e_temporal.py:5 | ControlFlowNode for Attribute | code/e_temporal.py:12 from import | list object | builtin-class list | @@ -466,12 +478,10 @@ | m_attributes.py:12 | ControlFlowNode for Attribute() | import | None | builtin-class NoneType | | m_attributes.py:12 | ControlFlowNode for C | import | class C | builtin-class type | | m_attributes.py:12 | ControlFlowNode for C() | import | C() | class C | -| m_attributes.py:12 | ControlFlowNode for C() | import | C() | class C | | m_attributes.py:13 | ControlFlowNode for Attribute | import | Method(Function C.foo, C()) | builtin-class method | | m_attributes.py:13 | ControlFlowNode for Attribute() | import | None | builtin-class NoneType | | m_attributes.py:13 | ControlFlowNode for C | import | class C | builtin-class type | | m_attributes.py:13 | ControlFlowNode for C() | import | C() | class C | -| m_attributes.py:13 | ControlFlowNode for C() | import | C() | class C | | m_attributes.py:13 | ControlFlowNode for IntegerLiteral | import | int 100 | builtin-class int | | n_nesting.py:8 | ControlFlowNode for FunctionExpr | import | Function foo | builtin-class function | | n_nesting.py:8 | ControlFlowNode for True | import | bool True | builtin-class bool | @@ -484,7 +494,6 @@ | n_nesting.py:15 | ControlFlowNode for Dict | runtime | Dict | builtin-class dict | | n_nesting.py:16 | ControlFlowNode for Str | runtime | 'inner' | builtin-class str | | n_nesting.py:16 | ControlFlowNode for inner | runtime | Function foo.inner | builtin-class function | -| n_nesting.py:16 | ControlFlowNode for inner | runtime | Function foo.inner | builtin-class function | | n_nesting.py:18 | ControlFlowNode for attrs | runtime | Dict | builtin-class dict | | n_nesting.py:22 | ControlFlowNode for FunctionExpr | import | Function f1 | builtin-class function | | n_nesting.py:23 | ControlFlowNode for C | code/n_nesting.py:25 from code/i_imports.py:38 from import | int 1 | builtin-class int | @@ -772,12 +781,12 @@ | r_regressions.py:90 | ControlFlowNode for TestFirst | import | class TestFirst | builtin-class type | | r_regressions.py:90 | ControlFlowNode for TestFirst() | import | TestFirst() | class TestFirst | | r_regressions.py:93 | ControlFlowNode for ImportExpr | import | Module sys | builtin-class module | -| r_regressions.py:95 | ControlFlowNode for Attribute | import | ('_ast', '_bisect', '_codecs', '_collections', '_datetime', '_elementtree', '_functools', '_heapq', '_imp', '_io', '_locale', '_md5', '_operator', '_pickle', '_posixsubprocess', '_random', '_sha1', '_sha256', '_sha512', '_signal', '_socket', '_sre', '_stat', '_string', '_struct', '_symtable', '_thread', '_tracemalloc', '_warnings', '_weakref', 'array', 'atexit', 'binascii', 'builtins', 'errno', 'faulthandler', 'fcntl', 'gc', 'grp', 'itertools', 'marshal', 'math', 'posix', 'pwd', 'pyexpat', 'select', 'spwd', 'sys', 'syslog', 'time', 'unicodedata', 'xxsubtype', 'zipimport', 'zlib') | builtin-class tuple | +| r_regressions.py:95 | ControlFlowNode for Attribute | import | (..., ...) | builtin-class tuple | | r_regressions.py:95 | ControlFlowNode for sys | import | Module sys | builtin-class module | | r_regressions.py:97 | ControlFlowNode for Compare | import | bool False | builtin-class bool | | r_regressions.py:97 | ControlFlowNode for Compare | import | bool True | builtin-class bool | | r_regressions.py:97 | ControlFlowNode for Str | import | 'time' | builtin-class str | -| r_regressions.py:97 | ControlFlowNode for _names | import | ('_ast', '_bisect', '_codecs', '_collections', '_datetime', '_elementtree', '_functools', '_heapq', '_imp', '_io', '_locale', '_md5', '_operator', '_pickle', '_posixsubprocess', '_random', '_sha1', '_sha256', '_sha512', '_signal', '_socket', '_sre', '_stat', '_string', '_struct', '_symtable', '_thread', '_tracemalloc', '_warnings', '_weakref', 'array', 'atexit', 'binascii', 'builtins', 'errno', 'faulthandler', 'fcntl', 'gc', 'grp', 'itertools', 'marshal', 'math', 'posix', 'pwd', 'pyexpat', 'select', 'spwd', 'sys', 'syslog', 'time', 'unicodedata', 'xxsubtype', 'zipimport', 'zlib') | builtin-class tuple | +| r_regressions.py:97 | ControlFlowNode for _names | import | (..., ...) | builtin-class tuple | | r_regressions.py:98 | ControlFlowNode for ImportExpr | import | Module time | builtin-class module | | r_regressions.py:100 | ControlFlowNode for C | import | class C | builtin-class type | | r_regressions.py:100 | ControlFlowNode for C() | import | C() | class C | diff --git a/python/ql/test/library-tests/PointsTo/new/Values.ql b/python/ql/test/library-tests/PointsTo/new/Values.ql index 64adbab58e0b..8e6bd9c9f002 100644 --- a/python/ql/test/library-tests/PointsTo/new/Values.ql +++ b/python/ql/test/library-tests/PointsTo/new/Values.ql @@ -6,4 +6,4 @@ import Util from ControlFlowNode f, Context ctx, Value v, ControlFlowNode origin where f.pointsTo(ctx, v, origin) -select locate(f.getLocation(), "abeghijklmnpqrstu"), f.toString(), ctx, v, v.getClass() +select locate(f.getLocation(), "abeghijklmnpqrstu"), f.toString(), ctx, vrepr(v), vrepr(v.getClass()) From 76c43b491fc2a29914fcf0d5cde67a90dd63be0d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 10 May 2019 16:04:26 +0100 Subject: [PATCH 082/128] Python points-to: Improve handling of attributes. --- .../semmle/python/objects/ObjectInternal.qll | 4 ++ .../ql/src/semmle/python/objects/TObject.qll | 8 ++++ python/ql/src/semmle/python/pointsto/Base.qll | 1 + .../src/semmle/python/pointsto/PointsTo.qll | 47 +++++++++++++++++-- .../PointsTo/new/SsaAttr.expected | 21 ++------- .../library-tests/PointsTo/new/SsaAttr.ql | 8 ++-- 6 files changed, 63 insertions(+), 26 deletions(-) diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 23e420d8deda..4b477dffcef0 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -402,6 +402,10 @@ module ObjectInternal { result = TBuiltinClassObject(Builtin::special("StaticMethod")) } + ObjectInternal boundMethod() { + result = TBuiltinClassObject(Builtin::special("MethodType")) + } + ObjectInternal moduleType() { result = TBuiltinClassObject(Builtin::special("ModuleType")) } diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 041551344176..1e80dbc53d3b 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -232,6 +232,14 @@ predicate call2(CallNode call, ControlFlowNode func, ControlFlowNode arg0, Contr arg1 = call.getArg(1) } +predicate call3(CallNode call, ControlFlowNode func, ControlFlowNode arg0, ControlFlowNode arg1, ControlFlowNode arg2) { + not exists(call.getArg(3)) and + func = call.getFunction() and + arg0 = call.getArg(0) and + arg1 = call.getArg(1) and + arg2 = call.getArg(2) +} + bindingset[self, function] predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext context) { exists(ObjectInternal obj, string name | diff --git a/python/ql/src/semmle/python/pointsto/Base.qll b/python/ql/src/semmle/python/pointsto/Base.qll index 74d68b3fc42d..782163c08426 100644 --- a/python/ql/src/semmle/python/pointsto/Base.qll +++ b/python/ql/src/semmle/python/pointsto/Base.qll @@ -378,6 +378,7 @@ class ArgumentRefinement extends PyNodeRefinement { ControlFlowNode getArgument() { result = argument } + CallNode getCall() { result = this.getDefiningNode() } } /** Deletion of an attribute `del obj.attr`. */ diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 248b641242c6..c6cdaad412ac 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -872,7 +872,7 @@ module InterProceduralPointsTo { ) } - predicate selfMethodCall(MethodCallsiteRefinement def, PointsToContext caller, Function func, PointsToContext callee) { + predicate selfMethodCall(SelfCallsiteRefinement def, PointsToContext caller, Function func, PointsToContext callee) { def.getInput().getSourceVariable().(Variable).isSelf() and exists(PythonFunctionObjectInternal method, CallNode call | method.getScope() = func and @@ -1232,6 +1232,15 @@ module Expressions { ) } + pragma [noinline] + predicate setattr_call(CallNode call, PointsToContext context, ControlFlowNode obj, string name, ObjectInternal val, ControlFlowNode origin) { + exists(ControlFlowNode arg1, ControlFlowNode arg2 | + call_to_setattr(call, context, obj, arg1, arg2) and + PointsToInternal::pointsTo(arg2, context, val, origin) and + PointsToInternal::pointsToString(arg1, context, name) + ) + } + pragma[noinline] private predicate call_to_getattr(ControlFlowNode call, PointsToContext context, ControlFlowNode arg0, ControlFlowNode arg1) { exists(ControlFlowNode func | @@ -1240,6 +1249,14 @@ module Expressions { ) } + pragma[noinline] + private predicate call_to_setattr(ControlFlowNode call, PointsToContext context, ControlFlowNode arg0, ControlFlowNode arg1, ControlFlowNode arg2) { + exists(ControlFlowNode func | + call3(call, func, arg0, arg1, arg2) and + PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("setattr"), _) + ) + } + pragma [noinline] private boolean otherComparisonEvaluatesTo(CompareNode comp, PointsToContext context, ControlFlowNode operand, ObjectInternal opvalue) { exists(Cmpop op | @@ -1935,6 +1952,7 @@ cached module Types { module AttributePointsTo { predicate pointsTo(AttrNode f, Context context, ObjectInternal value, ControlFlowNode origin) { + f.isLoad() and exists(EssaVariable var, string name, CfgOrigin orig | var.getASourceUse() = f.getObject(name) and variableAttributePointsTo(var, context, name, value, orig) and @@ -1961,6 +1979,8 @@ module AttributePointsTo { selfParameterAttributePointsTo(def, context, name, value, origin) or selfMethodCallsitePointsTo(def, context, name, value, origin) + or + argumentRefinementPointsTo(def, context, name, value, origin) } pragma [noinline] @@ -2002,7 +2022,7 @@ module AttributePointsTo { } private predicate selfParameterAttributePointsTo(ParameterDefinition def, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { - exists(MethodCallsiteRefinement call, Function func, PointsToContext caller | + exists(SelfCallsiteRefinement call, Function func, PointsToContext caller | InterProceduralPointsTo::selfMethodCall(call, caller, func, context) and def.isSelf() and def.getScope() = func and variableAttributePointsTo(call.getInput(), caller, name, value, origin) @@ -2010,14 +2030,33 @@ module AttributePointsTo { } /** Pass through for `self` for the implicit re-definition of `self` in `self.foo()`. */ - private predicate selfMethodCallsitePointsTo(MethodCallsiteRefinement def, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { + private predicate selfMethodCallsitePointsTo(SelfCallsiteRefinement def, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { /* The value of self remains the same, only the attributes may change */ exists(Function func, PointsToContext callee, EssaVariable exit_self | InterProceduralPointsTo::selfMethodCall(def, context, func, callee) and exit_self.getSourceVariable().(Variable).isSelf() and exit_self.getScope() = func and BaseFlow::reaches_exit(exit_self) and - variableAttributePointsTo(exit_self, context, name, value, origin) + variableAttributePointsTo(exit_self, callee, name, value, origin) + ) + } + + private predicate argumentRefinementPointsTo(ArgumentRefinement def, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { + exists(ObjectInternal callable | + PointsToInternal::pointsTo(def.getCall().getFunction(), context, callable, _) and + callable != ObjectInternal::builtin("setattr") + ) and + variableAttributePointsTo(def.getInput(), context, name, value, origin) + or + exists(string othername | + Expressions::setattr_call(def.getCall(), context, def.getInput().getASourceUse(), othername, _, _) and + not othername = name + ) and + variableAttributePointsTo(def.getInput(), context, name, value, origin) + or + exists(ControlFlowNode orig | + Expressions::setattr_call(def.getCall(), context, def.getInput().getASourceUse(), name, value, orig) and + origin = CfgOrigin::fromCfgNode(orig) ) } diff --git a/python/ql/test/library-tests/PointsTo/new/SsaAttr.expected b/python/ql/test/library-tests/PointsTo/new/SsaAttr.expected index 212abaab73d3..db5d628c4ba6 100644 --- a/python/ql/test/library-tests/PointsTo/new/SsaAttr.expected +++ b/python/ql/test/library-tests/PointsTo/new/SsaAttr.expected @@ -2,8 +2,10 @@ | b_condition.py:43 | v2_3 | x | Pi(v2_2) [true] | int 1 | import | | b_condition.py:47 | v2_4 | x | Pi(v2_2) [false] | int 1 | import | | b_condition.py:47 | v2_5 | x | phi(v2_3, v2_4) | int 1 | import | +| f_finally.py:3 | self_3 | _close | phi(self_1, self_2) | None | runtime | | f_finally.py:3 | self_3 | _closed | phi(self_1, self_2) | bool True | runtime | | f_finally.py:4 | self_1 | _closed | AttributeAssignment '_closed'(self_0) | bool True | runtime | +| f_finally.py:10 | self_2 | _close | AttributeAssignment '_close'(self_1) | None | runtime | | f_finally.py:10 | self_2 | _closed | AttributeAssignment '_close'(self_1) | bool True | runtime | | g_class_init.py:6 | self_1 | y | SelfCallsiteRefinement(self_0) | int 2 | runtime | | g_class_init.py:6 | self_1 | z | SelfCallsiteRefinement(self_0) | int 3 | runtime | @@ -16,27 +18,10 @@ | g_class_init.py:13 | self_0 | y | ParameterDefinition | int 2 | code/g_class_init.py:11 from code/g_class_init.py:6 from runtime | | g_class_init.py:14 | self_1 | y | AttributeAssignment 'z'(self_0) | int 2 | code/g_class_init.py:11 from code/g_class_init.py:6 from runtime | | g_class_init.py:14 | self_1 | z | AttributeAssignment 'z'(self_0) | int 3 | code/g_class_init.py:11 from code/g_class_init.py:6 from runtime | -| g_class_init.py:16 | self_0 | x | ParameterDefinition | int 1 | runtime | -| g_class_init.py:16 | self_0 | y | ParameterDefinition | int 2 | runtime | -| g_class_init.py:16 | self_0 | z | ParameterDefinition | int 3 | runtime | -| g_class_init.py:19 | self_1 | x | Pi(self_0) [true] | int 1 | runtime | -| g_class_init.py:19 | self_1 | y | Pi(self_0) [true] | int 2 | runtime | -| g_class_init.py:19 | self_1 | z | Pi(self_0) [true] | int 3 | runtime | -| g_class_init.py:20 | self_2 | x | Pi(self_0) [false] | int 1 | runtime | -| g_class_init.py:20 | self_2 | z | Pi(self_0) [false] | int 3 | runtime | -| g_class_init.py:20 | self_3 | x | phi(self_1, self_2) | int 1 | runtime | -| g_class_init.py:20 | self_3 | y | phi(self_1, self_2) | int 2 | runtime | -| g_class_init.py:20 | self_3 | z | phi(self_1, self_2) | int 3 | runtime | | g_class_init.py:46 | self_3 | version | phi(self_1, self_2) | 'v2' | runtime | | g_class_init.py:46 | self_3 | version | phi(self_1, self_2) | 'v3' | runtime | | g_class_init.py:48 | self_1 | version | AttributeAssignment 'version'(self_0) | 'v2' | runtime | | g_class_init.py:50 | self_2 | version | AttributeAssignment 'version'(self_0) | 'v3' | runtime | -| g_class_init.py:52 | self_0 | version | ParameterDefinition | 'v2' | runtime | -| g_class_init.py:52 | self_0 | version | ParameterDefinition | 'v3' | runtime | -| g_class_init.py:52 | self_2 | version | Pi(self_0) [false] | 'v3' | runtime | -| g_class_init.py:52 | self_3 | version | phi(self_1, self_2) | 'v2' | runtime | -| g_class_init.py:52 | self_3 | version | phi(self_1, self_2) | 'v3' | runtime | -| g_class_init.py:54 | self_1 | version | Pi(self_0) [true] | 'v2' | runtime | | k_getsetattr.py:6 | self_0 | a | ParameterDefinition | float 7.0 | code/k_getsetattr.py:15 from runtime | | k_getsetattr.py:6 | self_0 | c | ParameterDefinition | int 2 | code/k_getsetattr.py:15 from runtime | | k_getsetattr.py:7 | self_1 | a | ArgumentRefinement(self_0) | int 0 | code/k_getsetattr.py:15 from runtime | @@ -76,6 +61,6 @@ | k_getsetattr.py:27 | c2_1 | a | AttributeAssignment 'a'(c2_0) | int 20 | runtime | | k_getsetattr.py:28 | c2_2 | a | phi(c2_0, c2_1) | int 20 | runtime | | k_getsetattr.py:31 | c3_1 | a | AttributeAssignment 'a'(c3_0) | int 30 | runtime | +| m_attributes.py:6 | self_1 | a | AttributeAssignment 'a'(self_0) | Unknown value | runtime | | m_attributes.py:6 | self_1 | a | AttributeAssignment 'a'(self_0) | int 17 | runtime | | m_attributes.py:6 | self_1 | a | AttributeAssignment 'a'(self_0) | int 100 | code/m_attributes.py:13 from import | -| m_attributes.py:8 | self_0 | a | ParameterDefinition | int 17 | runtime | diff --git a/python/ql/test/library-tests/PointsTo/new/SsaAttr.ql b/python/ql/test/library-tests/PointsTo/new/SsaAttr.ql index 2a44c56ab1f2..4a4d83160c33 100644 --- a/python/ql/test/library-tests/PointsTo/new/SsaAttr.ql +++ b/python/ql/test/library-tests/PointsTo/new/SsaAttr.ql @@ -1,12 +1,12 @@ import python private import semmle.python.pointsto.PointsTo -private import semmle.python.pointsto.PointsToContext +private import semmle.python.objects.ObjectInternal import Util -from EssaVariable var, string name, Object o, PointsToContext ctx -where PointsTo::Test::ssa_variable_named_attribute_points_to(var, ctx, name, o, _, _) and not var.getSourceVariable() instanceof SpecialSsaSourceVariable +from EssaVariable var, string name, ObjectInternal o, Context ctx +where AttributePointsTo::variableAttributePointsTo(var, ctx, name, o, _) and not var.getSourceVariable() instanceof SpecialSsaSourceVariable select locate(var.getDefinition().getLocation(), "abdfgikm"), var.getRepresentation(), -name, var.getDefinition().getRepresentation(), repr(o), ctx +name, var.getDefinition().getRepresentation(), o, ctx From 8b2810567746986ca7cbe9e5d1c68ec35bb7f821 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 10 May 2019 16:05:37 +0100 Subject: [PATCH 083/128] Python points-to: update some more test results. --- .../PointsTo/new/TestEvaluate.expected | 12 ++---------- .../ql/test/library-tests/PointsTo/new/Util.qll | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/python/ql/test/library-tests/PointsTo/new/TestEvaluate.expected b/python/ql/test/library-tests/PointsTo/new/TestEvaluate.expected index a84466ea16a6..bb2bca055a3f 100644 --- a/python/ql/test/library-tests/PointsTo/new/TestEvaluate.expected +++ b/python/ql/test/library-tests/PointsTo/new/TestEvaluate.expected @@ -1,3 +1,5 @@ +WARNING: Predicate getSource has been deprecated and may be removed in future (TestEvaluate.ql:14,16-25) +WARNING: Predicate getSource has been deprecated and may be removed in future (TestEvaluate.ql:16,20-29) | b_condition.py:7 | Compare | true | x | NoneType None | | b_condition.py:13 | Compare | false | x | NoneType None | | b_condition.py:19 | UnaryExpr | true | x | NoneType None | @@ -33,16 +35,6 @@ | c_tests.py:26 | Compare | true | x | Tuple | | c_tests.py:29 | isinstance() | false | x | List | | c_tests.py:29 | isinstance() | true | x | Tuple | -| c_tests.py:34 | Compare | true | Attribute | NoneType None | -| c_tests.py:39 | Attribute | false | Attribute | int 0 | -| c_tests.py:39 | Attribute | true | Attribute | int 1 | -| c_tests.py:44 | Compare | false | Attribute | int 1 | -| c_tests.py:44 | Compare | true | Attribute | int 0 | -| c_tests.py:50 | isinstance() | false | Attribute | List | -| c_tests.py:50 | isinstance() | true | Attribute | Tuple | -| c_tests.py:53 | Compare | false | Attribute | Tuple | -| c_tests.py:53 | Compare | true | Attribute | List | -| c_tests.py:53 | Compare | true | Attribute | Tuple | | c_tests.py:60 | issubclass() | false | x | builtin-class type | | c_tests.py:60 | issubclass() | true | x | builtin-class bool | | c_tests.py:65 | hasattr() | false | x | builtin-class float | diff --git a/python/ql/test/library-tests/PointsTo/new/Util.qll b/python/ql/test/library-tests/PointsTo/new/Util.qll index 8e1d317cc68b..af8645f8d946 100644 --- a/python/ql/test/library-tests/PointsTo/new/Util.qll +++ b/python/ql/test/library-tests/PointsTo/new/Util.qll @@ -1,4 +1,5 @@ import python +import semmle.python.objects.ObjectInternal bindingset[which] string locate(Location l, string which) { @@ -28,3 +29,17 @@ string repr(Object o) { or o = theBoundMethodType() and result = "builtin-class method" } + +predicate long_tuple(Value v) { + v.(TupleObjectInternal).length() > 3 +} + +string vrepr(Value v) { + /* Work around differing names in 2/3 */ + not v = ObjectInternal::boundMethod() and + not long_tuple(v) and result = v.toString() + or + v = ObjectInternal::boundMethod() and result = "builtin-class method" + or + long_tuple(v) and result = "(..., ...)" +} \ No newline at end of file From a1bf071f554a9040b17c60c4084b3bea7bc5ee0f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 10 May 2019 16:36:09 +0100 Subject: [PATCH 084/128] Python points-to: Fix getattr() points-to. --- python/ql/src/semmle/python/pointsto/PointsTo.qll | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index c6cdaad412ac..d1216272a21f 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1951,10 +1951,13 @@ cached module Types { module AttributePointsTo { - predicate pointsTo(AttrNode f, Context context, ObjectInternal value, ControlFlowNode origin) { - f.isLoad() and + predicate pointsTo(ControlFlowNode f, Context context, ObjectInternal value, ControlFlowNode origin) { exists(EssaVariable var, string name, CfgOrigin orig | - var.getASourceUse() = f.getObject(name) and + f.isLoad() and + var.getASourceUse() = f.(AttrNode).getObject(name) + or + Expressions::getattr_call(f, var.getASourceUse(), context, _, name) + | variableAttributePointsTo(var, context, name, value, orig) and origin = orig.asCfgNodeOrHere(f) ) From 80f6883304f257bd19f90e8dc0c4226c5cbfd606 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 10 May 2019 16:40:53 +0100 Subject: [PATCH 085/128] Python points-to: Update test results. --- .../PointsTo/new/PointsToWithContext.expected | 64 +++++++++++++------ .../PointsTo/new/PointsToWithType.expected | 31 +++++---- .../PointsTo/new/Values.expected | 6 ++ 3 files changed, 64 insertions(+), 37 deletions(-) diff --git a/python/ql/test/library-tests/PointsTo/new/PointsToWithContext.expected b/python/ql/test/library-tests/PointsTo/new/PointsToWithContext.expected index cd5e3cdfae5b..b51dcb8d6b35 100755 --- a/python/ql/test/library-tests/PointsTo/new/PointsToWithContext.expected +++ b/python/ql/test/library-tests/PointsTo/new/PointsToWithContext.expected @@ -1,3 +1,4 @@ +WARNING: Predicate points_to has been deprecated and may be removed in future (PointsToWithContext.ql:8,7-26) | a_simple.py:2 | ControlFlowNode for FloatLiteral | float 1.0 | builtin-class float | 2 | import | | a_simple.py:2 | ControlFlowNode for f1 | float 1.0 | builtin-class float | 2 | import | | a_simple.py:3 | ControlFlowNode for dict | builtin-class dict | builtin-class type | 3 | import | @@ -169,15 +170,11 @@ | b_condition.py:101 | ControlFlowNode for not_or_not | Function not_or_not | builtin-class function | 101 | import | | b_condition.py:102 | ControlFlowNode for Tuple | Tuple | builtin-class tuple | 102 | runtime | | b_condition.py:102 | ControlFlowNode for UnaryExpr | bool False | builtin-class bool | 102 | runtime | -| b_condition.py:102 | ControlFlowNode for UnaryExpr | bool True | builtin-class bool | 102 | runtime | | b_condition.py:102 | ControlFlowNode for a | a | builtin-class tuple | 101 | runtime | | b_condition.py:102 | ControlFlowNode for isinstance | Builtin-function isinstance | builtin-class builtin_function_or_method | 102 | runtime | -| b_condition.py:102 | ControlFlowNode for isinstance() | bool False | builtin-class bool | 102 | runtime | | b_condition.py:102 | ControlFlowNode for isinstance() | bool True | builtin-class bool | 102 | runtime | | b_condition.py:102 | ControlFlowNode for list | builtin-class list | builtin-class type | 102 | runtime | | b_condition.py:102 | ControlFlowNode for tuple | builtin-class tuple | builtin-class type | 102 | runtime | -| b_condition.py:103 | ControlFlowNode for TypeError | builtin-class TypeError | builtin-class type | 103 | runtime | -| b_condition.py:103 | ControlFlowNode for TypeError() | TypeError() | builtin-class TypeError | 103 | runtime | | b_condition.py:104 | ControlFlowNode for UnaryExpr | bool False | builtin-class bool | 104 | runtime | | b_condition.py:104 | ControlFlowNode for UnaryExpr | bool True | builtin-class bool | 104 | runtime | | b_condition.py:104 | ControlFlowNode for a | a | builtin-class tuple | 101 | runtime | @@ -221,6 +218,7 @@ | g_class_init.py:3 | ControlFlowNode for object | builtin-class object | builtin-class type | 3 | import | | g_class_init.py:5 | ControlFlowNode for FunctionExpr | Function __init__ | builtin-class function | 5 | import | | g_class_init.py:5 | ControlFlowNode for __init__ | Function __init__ | builtin-class function | 5 | import | +| g_class_init.py:6 | ControlFlowNode for Attribute | Attribute | builtin-class method | 6 | runtime | | g_class_init.py:6 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 9 | runtime | | g_class_init.py:6 | ControlFlowNode for self | self | class C | 5 | runtime | | g_class_init.py:7 | ControlFlowNode for Attribute | int 1 | builtin-class int | 7 | runtime | @@ -231,6 +229,7 @@ | g_class_init.py:10 | ControlFlowNode for Attribute | int 2 | builtin-class int | 10 | code/g_class_init.py:6 from runtime | | g_class_init.py:10 | ControlFlowNode for IntegerLiteral | int 2 | builtin-class int | 10 | code/g_class_init.py:6 from runtime | | g_class_init.py:10 | ControlFlowNode for self | self | class C | 5 | code/g_class_init.py:6 from runtime | +| g_class_init.py:11 | ControlFlowNode for Attribute | Attribute | builtin-class method | 11 | code/g_class_init.py:6 from runtime | | g_class_init.py:11 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 13 | code/g_class_init.py:6 from runtime | | g_class_init.py:11 | ControlFlowNode for self | self | class C | 5 | code/g_class_init.py:6 from runtime | | g_class_init.py:13 | ControlFlowNode for FunctionExpr | Function _init2 | builtin-class function | 13 | import | @@ -240,16 +239,13 @@ | g_class_init.py:14 | ControlFlowNode for self | self | class C | 5 | code/g_class_init.py:11 from code/g_class_init.py:6 from runtime | | g_class_init.py:16 | ControlFlowNode for FunctionExpr | Function method | builtin-class function | 16 | import | | g_class_init.py:16 | ControlFlowNode for method | Function method | builtin-class function | 16 | import | -| g_class_init.py:17 | ControlFlowNode for Attribute | int 1 | builtin-class int | 7 | runtime | | g_class_init.py:17 | ControlFlowNode for self | self | class C | 16 | runtime | -| g_class_init.py:18 | ControlFlowNode for Attribute | int 2 | builtin-class int | 10 | runtime | | g_class_init.py:18 | ControlFlowNode for int | builtin-class int | builtin-class type | 18 | runtime | | g_class_init.py:18 | ControlFlowNode for isinstance | Builtin-function isinstance | builtin-class builtin_function_or_method | 18 | runtime | +| g_class_init.py:18 | ControlFlowNode for isinstance() | bool False | builtin-class bool | 18 | runtime | | g_class_init.py:18 | ControlFlowNode for isinstance() | bool True | builtin-class bool | 18 | runtime | | g_class_init.py:18 | ControlFlowNode for self | self | class C | 16 | runtime | -| g_class_init.py:19 | ControlFlowNode for Attribute | int 2 | builtin-class int | 10 | runtime | | g_class_init.py:19 | ControlFlowNode for self | self | class C | 16 | runtime | -| g_class_init.py:20 | ControlFlowNode for Attribute | int 3 | builtin-class int | 14 | runtime | | g_class_init.py:20 | ControlFlowNode for self | self | class C | 16 | runtime | | g_class_init.py:24 | ControlFlowNode for ClassExpr | class Oddities | builtin-class type | 24 | import | | g_class_init.py:24 | ControlFlowNode for Oddities | class Oddities | builtin-class type | 24 | import | @@ -265,7 +261,6 @@ | g_class_init.py:32 | ControlFlowNode for object | builtin-class object | builtin-class type | 32 | import | | g_class_init.py:34 | ControlFlowNode for FunctionExpr | Function __init__ | builtin-class function | 34 | import | | g_class_init.py:34 | ControlFlowNode for __init__ | Function __init__ | builtin-class function | 34 | import | -| g_class_init.py:35 | ControlFlowNode for Attribute | super().x | builtin-class method | 35 | runtime | | g_class_init.py:35 | ControlFlowNode for D | class D | builtin-class type | 32 | runtime | | g_class_init.py:35 | ControlFlowNode for self | self | class D | 34 | runtime | | g_class_init.py:35 | ControlFlowNode for super | builtin-class super | builtin-class type | 35 | runtime | @@ -292,8 +287,6 @@ | g_class_init.py:50 | ControlFlowNode for self | self | class E | 46 | runtime | | g_class_init.py:52 | ControlFlowNode for FunctionExpr | Function meth | builtin-class function | 52 | import | | g_class_init.py:52 | ControlFlowNode for meth | Function meth | builtin-class function | 52 | import | -| g_class_init.py:53 | ControlFlowNode for Attribute | 'v2' | builtin-class str | 42 | runtime | -| g_class_init.py:53 | ControlFlowNode for Attribute | 'v3' | builtin-class str | 43 | runtime | | g_class_init.py:53 | ControlFlowNode for Compare | bool False | builtin-class bool | 53 | runtime | | g_class_init.py:53 | ControlFlowNode for Compare | bool True | builtin-class bool | 53 | runtime | | g_class_init.py:53 | ControlFlowNode for V2 | 'v2' | builtin-class str | 42 | runtime | @@ -325,6 +318,7 @@ | h_classes.py:12 | ControlFlowNode for Tuple | Tuple | builtin-class tuple | 12 | import | | h_classes.py:12 | ControlFlowNode for object | builtin-class object | builtin-class type | 12 | import | | h_classes.py:12 | ControlFlowNode for type | builtin-class type | builtin-class type | 12 | import | +| h_classes.py:12 | ControlFlowNode for type() | type() | builtin-class type | 12 | import | | h_classes.py:14 | ControlFlowNode for FunctionExpr | Function k | builtin-class function | 14 | import | | h_classes.py:14 | ControlFlowNode for k | Function k | builtin-class function | 14 | import | | h_classes.py:15 | ControlFlowNode for C | class C | builtin-class type | 3 | runtime | @@ -335,29 +329,42 @@ | h_classes.py:16 | ControlFlowNode for type | builtin-class type | builtin-class type | 16 | runtime | | h_classes.py:16 | ControlFlowNode for type() | builtin-class module | builtin-class type | 16 | runtime | | h_classes.py:17 | ControlFlowNode for type | builtin-class type | builtin-class type | 17 | runtime | -| h_classes.py:17 | ControlFlowNode for type() | *UNKNOWN TYPE* | builtin-class type | 17 | runtime | +| h_classes.py:17 | ControlFlowNode for type() | *UNKNOWN TYPE* | *UNKNOWN TYPE* | 17 | runtime | | h_classes.py:18 | ControlFlowNode for Dict | Dict | builtin-class dict | 18 | runtime | | h_classes.py:18 | ControlFlowNode for Tuple | Tuple | builtin-class tuple | 18 | runtime | | h_classes.py:18 | ControlFlowNode for object | builtin-class object | builtin-class type | 18 | runtime | | h_classes.py:18 | ControlFlowNode for type | builtin-class type | builtin-class type | 18 | runtime | +| h_classes.py:18 | ControlFlowNode for type() | type() | builtin-class type | 18 | runtime | | h_classes.py:23 | ControlFlowNode for Base | class Base | builtin-class type | 23 | import | | h_classes.py:23 | ControlFlowNode for ClassExpr | class Base | builtin-class type | 23 | import | | h_classes.py:23 | ControlFlowNode for object | builtin-class object | builtin-class type | 23 | import | | h_classes.py:25 | ControlFlowNode for FunctionExpr | Function __init__ | builtin-class function | 25 | import | | h_classes.py:25 | ControlFlowNode for __init__ | Function __init__ | builtin-class function | 25 | import | +| h_classes.py:26 | ControlFlowNode for Compare | bool False | builtin-class bool | 26 | code/h_classes.py:42 from import | | h_classes.py:26 | ControlFlowNode for Compare | bool False | builtin-class bool | 26 | runtime | +| h_classes.py:26 | ControlFlowNode for Compare | bool True | builtin-class bool | 26 | code/h_classes.py:42 from import | | h_classes.py:26 | ControlFlowNode for Compare | bool True | builtin-class bool | 26 | runtime | +| h_classes.py:26 | ControlFlowNode for IntegerLiteral | int 1 | builtin-class int | 26 | code/h_classes.py:42 from import | | h_classes.py:26 | ControlFlowNode for IntegerLiteral | int 1 | builtin-class int | 26 | runtime | +| h_classes.py:27 | ControlFlowNode for Attribute | class Derived1 | builtin-class type | 33 | code/h_classes.py:42 from import | | h_classes.py:27 | ControlFlowNode for Attribute | class Derived1 | builtin-class type | 33 | runtime | +| h_classes.py:27 | ControlFlowNode for Derived1 | class Derived1 | builtin-class type | 33 | code/h_classes.py:42 from import | | h_classes.py:27 | ControlFlowNode for Derived1 | class Derived1 | builtin-class type | 33 | runtime | | h_classes.py:27 | ControlFlowNode for self | self | class Base | 25 | runtime | +| h_classes.py:28 | ControlFlowNode for Compare | bool False | builtin-class bool | 28 | code/h_classes.py:42 from import | | h_classes.py:28 | ControlFlowNode for Compare | bool False | builtin-class bool | 28 | runtime | +| h_classes.py:28 | ControlFlowNode for Compare | bool True | builtin-class bool | 28 | code/h_classes.py:42 from import | | h_classes.py:28 | ControlFlowNode for Compare | bool True | builtin-class bool | 28 | runtime | +| h_classes.py:28 | ControlFlowNode for IntegerLiteral | int 2 | builtin-class int | 28 | code/h_classes.py:42 from import | | h_classes.py:28 | ControlFlowNode for IntegerLiteral | int 2 | builtin-class int | 28 | runtime | +| h_classes.py:29 | ControlFlowNode for Attribute | class Derived2 | builtin-class type | 36 | code/h_classes.py:42 from import | | h_classes.py:29 | ControlFlowNode for Attribute | class Derived2 | builtin-class type | 36 | runtime | +| h_classes.py:29 | ControlFlowNode for Derived2 | class Derived2 | builtin-class type | 36 | code/h_classes.py:42 from import | | h_classes.py:29 | ControlFlowNode for Derived2 | class Derived2 | builtin-class type | 36 | runtime | | h_classes.py:29 | ControlFlowNode for self | self | class Base | 25 | runtime | +| h_classes.py:31 | ControlFlowNode for Attribute | class Derived3 | builtin-class type | 39 | code/h_classes.py:42 from import | | h_classes.py:31 | ControlFlowNode for Attribute | class Derived3 | builtin-class type | 39 | runtime | +| h_classes.py:31 | ControlFlowNode for Derived3 | class Derived3 | builtin-class type | 39 | code/h_classes.py:42 from import | | h_classes.py:31 | ControlFlowNode for Derived3 | class Derived3 | builtin-class type | 39 | runtime | | h_classes.py:31 | ControlFlowNode for self | self | class Base | 25 | runtime | | h_classes.py:33 | ControlFlowNode for Base | class Base | builtin-class type | 23 | import | @@ -503,6 +510,7 @@ | k_getsetattr.py:14 | ControlFlowNode for self | self | class C | 12 | runtime | | k_getsetattr.py:14 | ControlFlowNode for setattr | Builtin-function setattr | builtin-class builtin_function_or_method | 14 | runtime | | k_getsetattr.py:14 | ControlFlowNode for setattr() | NoneType None | builtin-class NoneType | 14 | runtime | +| k_getsetattr.py:15 | ControlFlowNode for Attribute | Attribute | builtin-class method | 15 | runtime | | k_getsetattr.py:15 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 6 | runtime | | k_getsetattr.py:15 | ControlFlowNode for self | self | class C | 12 | runtime | | k_getsetattr.py:16 | ControlFlowNode for Str | 'a' | builtin-class str | 16 | runtime | @@ -545,6 +553,8 @@ | l_calls.py:3 | ControlFlowNode for FunctionExpr | Function foo | builtin-class function | 3 | import | | l_calls.py:3 | ControlFlowNode for List | List | builtin-class list | 3 | import | | l_calls.py:3 | ControlFlowNode for foo | Function foo | builtin-class function | 3 | import | +| l_calls.py:4 | ControlFlowNode for Attribute | Attribute | builtin-class method | 4 | code/l_calls.py:9 from import | +| l_calls.py:4 | ControlFlowNode for Attribute | Attribute | builtin-class method | 4 | runtime | | l_calls.py:4 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 4 | code/l_calls.py:9 from import | | l_calls.py:4 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 4 | runtime | | l_calls.py:4 | ControlFlowNode for Str | 'x' | builtin-class str | 4 | code/l_calls.py:9 from import | @@ -571,7 +581,7 @@ | l_calls.py:14 | ControlFlowNode for classmethod() | classmethod() | builtin-class classmethod | 14 | import | | l_calls.py:15 | ControlFlowNode for FunctionExpr | Function cm | builtin-class function | 15 | import | | l_calls.py:15 | ControlFlowNode for cm | classmethod() | builtin-class classmethod | 14 | import | -| l_calls.py:16 | ControlFlowNode for cls | class Owner | builtin-class type | 23 | code/l_calls.py:24 from runtime | +| l_calls.py:16 | ControlFlowNode for cls | class Owner | builtin-class type | 12 | code/l_calls.py:24 from runtime | | l_calls.py:18 | ControlFlowNode for classmethod | builtin-class classmethod | builtin-class type | 18 | import | | l_calls.py:18 | ControlFlowNode for classmethod() | classmethod() | builtin-class classmethod | 18 | import | | l_calls.py:19 | ControlFlowNode for FunctionExpr | Function cm2 | builtin-class function | 19 | import | @@ -579,13 +589,15 @@ | l_calls.py:20 | ControlFlowNode for arg | int 1 | builtin-class int | 25 | code/l_calls.py:25 from runtime | | l_calls.py:23 | ControlFlowNode for FunctionExpr | Function m | builtin-class function | 23 | import | | l_calls.py:23 | ControlFlowNode for m | Function m | builtin-class function | 23 | import | -| l_calls.py:24 | ControlFlowNode for Attribute() | class Owner | builtin-class type | 23 | runtime | +| l_calls.py:24 | ControlFlowNode for Attribute | Attribute | builtin-class method | 24 | runtime | +| l_calls.py:24 | ControlFlowNode for Attribute() | class Owner | builtin-class type | 12 | runtime | | l_calls.py:24 | ControlFlowNode for IntegerLiteral | int 0 | builtin-class int | 24 | runtime | -| l_calls.py:24 | ControlFlowNode for a | class Owner | builtin-class type | 23 | runtime | +| l_calls.py:24 | ControlFlowNode for a | class Owner | builtin-class type | 12 | runtime | | l_calls.py:24 | ControlFlowNode for self | self | class Owner | 23 | runtime | +| l_calls.py:25 | ControlFlowNode for Attribute | Attribute | builtin-class method | 25 | runtime | | l_calls.py:25 | ControlFlowNode for Attribute() | int 1 | builtin-class int | 25 | runtime | | l_calls.py:25 | ControlFlowNode for IntegerLiteral | int 1 | builtin-class int | 25 | runtime | -| l_calls.py:25 | ControlFlowNode for a | class Owner | builtin-class type | 23 | runtime | +| l_calls.py:25 | ControlFlowNode for a | class Owner | builtin-class type | 12 | runtime | | m_attributes.py:3 | ControlFlowNode for C | class C | builtin-class type | 3 | import | | m_attributes.py:3 | ControlFlowNode for ClassExpr | class C | builtin-class type | 3 | import | | m_attributes.py:3 | ControlFlowNode for object | builtin-class object | builtin-class type | 3 | import | @@ -599,13 +611,17 @@ | m_attributes.py:6 | ControlFlowNode for self | self | class C | 5 | runtime | | m_attributes.py:8 | ControlFlowNode for FunctionExpr | Function foo | builtin-class function | 8 | import | | m_attributes.py:8 | ControlFlowNode for foo | Function foo | builtin-class function | 8 | import | -| m_attributes.py:9 | ControlFlowNode for Attribute | int 17 | builtin-class int | 5 | runtime | +| m_attributes.py:9 | ControlFlowNode for self | C() | class C | 12 | code/m_attributes.py:12 from import | +| m_attributes.py:9 | ControlFlowNode for self | C() | class C | 13 | code/m_attributes.py:13 from import | | m_attributes.py:9 | ControlFlowNode for self | self | class C | 8 | runtime | +| m_attributes.py:10 | ControlFlowNode for Attribute | int 100 | builtin-class int | 13 | code/m_attributes.py:13 from import | | m_attributes.py:10 | ControlFlowNode for other | C() | class C | 12 | code/m_attributes.py:12 from import | | m_attributes.py:10 | ControlFlowNode for other | C() | class C | 13 | code/m_attributes.py:13 from import | +| m_attributes.py:12 | ControlFlowNode for Attribute | Attribute | builtin-class method | 12 | import | | m_attributes.py:12 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 8 | import | | m_attributes.py:12 | ControlFlowNode for C | class C | builtin-class type | 3 | import | | m_attributes.py:12 | ControlFlowNode for C() | C() | class C | 12 | import | +| m_attributes.py:13 | ControlFlowNode for Attribute | Attribute | builtin-class method | 13 | import | | m_attributes.py:13 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 8 | import | | m_attributes.py:13 | ControlFlowNode for C | class C | builtin-class type | 3 | import | | m_attributes.py:13 | ControlFlowNode for C() | C() | class C | 13 | import | @@ -902,6 +918,7 @@ | r_regressions.py:5 | ControlFlowNode for object | builtin-class object | builtin-class type | 5 | import | | r_regressions.py:7 | ControlFlowNode for FunctionExpr | Function __init__ | builtin-class function | 7 | import | | r_regressions.py:7 | ControlFlowNode for __init__ | Function __init__ | builtin-class function | 7 | import | +| r_regressions.py:9 | ControlFlowNode for Attribute | Attribute | builtin-class method | 9 | runtime | | r_regressions.py:9 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 11 | runtime | | r_regressions.py:9 | ControlFlowNode for self | self | class Queue | 7 | runtime | | r_regressions.py:11 | ControlFlowNode for FunctionExpr | Function _after_fork | builtin-class function | 11 | import | @@ -918,10 +935,7 @@ | r_regressions.py:16 | ControlFlowNode for True | bool True | builtin-class bool | 16 | runtime | | r_regressions.py:16 | ControlFlowNode for self | self | class Queue | 15 | runtime | | r_regressions.py:18 | ControlFlowNode for self | self | class Queue | 15 | runtime | -| r_regressions.py:20 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 13 | runtime | -| r_regressions.py:20 | ControlFlowNode for close | NoneType None | builtin-class NoneType | 13 | runtime | | r_regressions.py:20 | ControlFlowNode for self | self | class Queue | 15 | runtime | -| r_regressions.py:21 | ControlFlowNode for close | NoneType None | builtin-class NoneType | 13 | runtime | | r_regressions.py:22 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 22 | runtime | | r_regressions.py:22 | ControlFlowNode for None | NoneType None | builtin-class NoneType | 22 | runtime | | r_regressions.py:22 | ControlFlowNode for self | self | class Queue | 15 | runtime | @@ -961,6 +975,8 @@ | r_regressions.py:63 | ControlFlowNode for is_class | bool True | builtin-class bool | 62 | code/r_regressions.py:85 from import | | r_regressions.py:68 | ControlFlowNode for FunctionExpr | Function _wrapper | builtin-class function | 68 | code/r_regressions.py:85 from import | | r_regressions.py:68 | ControlFlowNode for _wrapper | Function _wrapper | builtin-class function | 68 | code/r_regressions.py:85 from import | +| r_regressions.py:68 | ControlFlowNode for args | args | builtin-class tuple | 68 | runtime | +| r_regressions.py:68 | ControlFlowNode for kwargs | kwargs | builtin-class dict | 68 | runtime | | r_regressions.py:72 | ControlFlowNode for is_class | bool True | builtin-class bool | 62 | code/r_regressions.py:85 from import | | r_regressions.py:73 | ControlFlowNode for _wrapper | Function _wrapper | builtin-class function | 68 | code/r_regressions.py:85 from import | | r_regressions.py:73 | ControlFlowNode for obj | class TestFirst | builtin-class type | 86 | code/r_regressions.py:85 from import | @@ -973,6 +989,8 @@ | r_regressions.py:80 | ControlFlowNode for deco | Function deco | builtin-class function | 80 | import | | r_regressions.py:81 | ControlFlowNode for FunctionExpr | Function _wrapper | builtin-class function | 81 | runtime | | r_regressions.py:81 | ControlFlowNode for _wrapper | Function _wrapper | builtin-class function | 81 | runtime | +| r_regressions.py:81 | ControlFlowNode for args | args | builtin-class tuple | 81 | runtime | +| r_regressions.py:81 | ControlFlowNode for kwargs | kwargs | builtin-class dict | 81 | runtime | | r_regressions.py:83 | ControlFlowNode for _wrapper | Function _wrapper | builtin-class function | 81 | runtime | | r_regressions.py:85 | ControlFlowNode for Str | 'method' | builtin-class str | 85 | import | | r_regressions.py:85 | ControlFlowNode for deco | Function deco | builtin-class function | 80 | import | @@ -986,6 +1004,7 @@ | r_regressions.py:87 | ControlFlowNode for method | Function method | builtin-class function | 87 | import | | r_regressions.py:88 | ControlFlowNode for Str | 'hello world' | builtin-class str | 88 | code/r_regressions.py:90 from import | | r_regressions.py:88 | ControlFlowNode for Str | 'hello world' | builtin-class str | 88 | runtime | +| r_regressions.py:90 | ControlFlowNode for Attribute | Attribute | builtin-class method | 90 | import | | r_regressions.py:90 | ControlFlowNode for Attribute() | 'hello world' | builtin-class str | 88 | import | | r_regressions.py:90 | ControlFlowNode for TestFirst | class TestFirst | builtin-class type | 86 | import | | r_regressions.py:90 | ControlFlowNode for TestFirst() | TestFirst() | class TestFirst | 90 | import | @@ -994,6 +1013,7 @@ | r_regressions.py:95 | ControlFlowNode for Attribute | tuple object | builtin-class tuple | 95 | import | | r_regressions.py:95 | ControlFlowNode for _names | tuple object | builtin-class tuple | 95 | import | | r_regressions.py:95 | ControlFlowNode for sys | Module sys | builtin-class module | 93 | import | +| r_regressions.py:97 | ControlFlowNode for Compare | bool False | builtin-class bool | 97 | import | | r_regressions.py:97 | ControlFlowNode for Compare | bool True | builtin-class bool | 97 | import | | r_regressions.py:97 | ControlFlowNode for Str | 'time' | builtin-class str | 97 | import | | r_regressions.py:97 | ControlFlowNode for _names | tuple object | builtin-class tuple | 95 | import | @@ -1058,6 +1078,7 @@ | t_type.py:10 | ControlFlowNode for Tuple | Tuple | builtin-class tuple | 10 | import | | t_type.py:10 | ControlFlowNode for object | builtin-class object | builtin-class type | 10 | import | | t_type.py:10 | ControlFlowNode for type | builtin-class type | builtin-class type | 10 | import | +| t_type.py:10 | ControlFlowNode for type() | type() | builtin-class type | 10 | import | | t_type.py:12 | ControlFlowNode for FunctionExpr | Function k | builtin-class function | 12 | import | | t_type.py:12 | ControlFlowNode for k | Function k | builtin-class function | 12 | import | | t_type.py:13 | ControlFlowNode for C | class C | builtin-class type | 3 | runtime | @@ -1068,11 +1089,12 @@ | t_type.py:14 | ControlFlowNode for type | builtin-class type | builtin-class type | 14 | runtime | | t_type.py:14 | ControlFlowNode for type() | builtin-class module | builtin-class type | 14 | runtime | | t_type.py:15 | ControlFlowNode for type | builtin-class type | builtin-class type | 15 | runtime | -| t_type.py:15 | ControlFlowNode for type() | *UNKNOWN TYPE* | builtin-class type | 15 | runtime | +| t_type.py:15 | ControlFlowNode for type() | *UNKNOWN TYPE* | *UNKNOWN TYPE* | 15 | runtime | | t_type.py:16 | ControlFlowNode for Dict | Dict | builtin-class dict | 16 | runtime | | t_type.py:16 | ControlFlowNode for Tuple | Tuple | builtin-class tuple | 16 | runtime | | t_type.py:16 | ControlFlowNode for object | builtin-class object | builtin-class type | 16 | runtime | | t_type.py:16 | ControlFlowNode for type | builtin-class type | builtin-class type | 16 | runtime | +| t_type.py:16 | ControlFlowNode for type() | type() | builtin-class type | 16 | runtime | | u_paired_values.py:2 | ControlFlowNode for FunctionExpr | Function return_if_true | builtin-class function | 2 | import | | u_paired_values.py:2 | ControlFlowNode for return_if_true | Function return_if_true | builtin-class function | 2 | import | | u_paired_values.py:3 | ControlFlowNode for cond | bool False | builtin-class bool | 8 | code/u_paired_values.py:8 from code/u_paired_values.py:14 from import | diff --git a/python/ql/test/library-tests/PointsTo/new/PointsToWithType.expected b/python/ql/test/library-tests/PointsTo/new/PointsToWithType.expected index 86e75f991cb1..999cebadfa7c 100644 --- a/python/ql/test/library-tests/PointsTo/new/PointsToWithType.expected +++ b/python/ql/test/library-tests/PointsTo/new/PointsToWithType.expected @@ -1,3 +1,4 @@ +WARNING: Predicate points_to has been deprecated and may be removed in future (PointsToWithType.ql:7,7-26) | a_simple.py:2 | ControlFlowNode for FloatLiteral | float 1.0 | builtin-class float | 2 | | a_simple.py:2 | ControlFlowNode for f1 | float 1.0 | builtin-class float | 2 | | a_simple.py:3 | ControlFlowNode for dict | builtin-class dict | builtin-class type | 3 | @@ -169,15 +170,11 @@ | b_condition.py:101 | ControlFlowNode for not_or_not | Function not_or_not | builtin-class function | 101 | | b_condition.py:102 | ControlFlowNode for Tuple | Tuple | builtin-class tuple | 102 | | b_condition.py:102 | ControlFlowNode for UnaryExpr | bool False | builtin-class bool | 102 | -| b_condition.py:102 | ControlFlowNode for UnaryExpr | bool True | builtin-class bool | 102 | | b_condition.py:102 | ControlFlowNode for a | a | builtin-class tuple | 101 | | b_condition.py:102 | ControlFlowNode for isinstance | Builtin-function isinstance | builtin-class builtin_function_or_method | 102 | -| b_condition.py:102 | ControlFlowNode for isinstance() | bool False | builtin-class bool | 102 | | b_condition.py:102 | ControlFlowNode for isinstance() | bool True | builtin-class bool | 102 | | b_condition.py:102 | ControlFlowNode for list | builtin-class list | builtin-class type | 102 | | b_condition.py:102 | ControlFlowNode for tuple | builtin-class tuple | builtin-class type | 102 | -| b_condition.py:103 | ControlFlowNode for TypeError | builtin-class TypeError | builtin-class type | 103 | -| b_condition.py:103 | ControlFlowNode for TypeError() | TypeError() | builtin-class TypeError | 103 | | b_condition.py:104 | ControlFlowNode for UnaryExpr | bool False | builtin-class bool | 104 | | b_condition.py:104 | ControlFlowNode for UnaryExpr | bool True | builtin-class bool | 104 | | b_condition.py:104 | ControlFlowNode for a | a | builtin-class tuple | 101 | @@ -334,6 +331,7 @@ | g_class_init.py:3 | ControlFlowNode for object | builtin-class object | builtin-class type | 3 | | g_class_init.py:5 | ControlFlowNode for FunctionExpr | Function __init__ | builtin-class function | 5 | | g_class_init.py:5 | ControlFlowNode for __init__ | Function __init__ | builtin-class function | 5 | +| g_class_init.py:6 | ControlFlowNode for Attribute | Attribute | builtin-class method | 6 | | g_class_init.py:6 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 9 | | g_class_init.py:6 | ControlFlowNode for self | self | class C | 5 | | g_class_init.py:7 | ControlFlowNode for Attribute | int 1 | builtin-class int | 7 | @@ -344,6 +342,7 @@ | g_class_init.py:10 | ControlFlowNode for Attribute | int 2 | builtin-class int | 10 | | g_class_init.py:10 | ControlFlowNode for IntegerLiteral | int 2 | builtin-class int | 10 | | g_class_init.py:10 | ControlFlowNode for self | self | class C | 5 | +| g_class_init.py:11 | ControlFlowNode for Attribute | Attribute | builtin-class method | 11 | | g_class_init.py:11 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 13 | | g_class_init.py:11 | ControlFlowNode for self | self | class C | 5 | | g_class_init.py:13 | ControlFlowNode for FunctionExpr | Function _init2 | builtin-class function | 13 | @@ -353,16 +352,13 @@ | g_class_init.py:14 | ControlFlowNode for self | self | class C | 5 | | g_class_init.py:16 | ControlFlowNode for FunctionExpr | Function method | builtin-class function | 16 | | g_class_init.py:16 | ControlFlowNode for method | Function method | builtin-class function | 16 | -| g_class_init.py:17 | ControlFlowNode for Attribute | int 1 | builtin-class int | 7 | | g_class_init.py:17 | ControlFlowNode for self | self | class C | 16 | -| g_class_init.py:18 | ControlFlowNode for Attribute | int 2 | builtin-class int | 10 | | g_class_init.py:18 | ControlFlowNode for int | builtin-class int | builtin-class type | 18 | | g_class_init.py:18 | ControlFlowNode for isinstance | Builtin-function isinstance | builtin-class builtin_function_or_method | 18 | +| g_class_init.py:18 | ControlFlowNode for isinstance() | bool False | builtin-class bool | 18 | | g_class_init.py:18 | ControlFlowNode for isinstance() | bool True | builtin-class bool | 18 | | g_class_init.py:18 | ControlFlowNode for self | self | class C | 16 | -| g_class_init.py:19 | ControlFlowNode for Attribute | int 2 | builtin-class int | 10 | | g_class_init.py:19 | ControlFlowNode for self | self | class C | 16 | -| g_class_init.py:20 | ControlFlowNode for Attribute | int 3 | builtin-class int | 14 | | g_class_init.py:20 | ControlFlowNode for self | self | class C | 16 | | g_class_init.py:24 | ControlFlowNode for ClassExpr | class Oddities | builtin-class type | 24 | | g_class_init.py:24 | ControlFlowNode for Oddities | class Oddities | builtin-class type | 24 | @@ -378,7 +374,6 @@ | g_class_init.py:32 | ControlFlowNode for object | builtin-class object | builtin-class type | 32 | | g_class_init.py:34 | ControlFlowNode for FunctionExpr | Function __init__ | builtin-class function | 34 | | g_class_init.py:34 | ControlFlowNode for __init__ | Function __init__ | builtin-class function | 34 | -| g_class_init.py:35 | ControlFlowNode for Attribute | super().x | builtin-class method | 35 | | g_class_init.py:35 | ControlFlowNode for D | class D | builtin-class type | 32 | | g_class_init.py:35 | ControlFlowNode for self | self | class D | 34 | | g_class_init.py:35 | ControlFlowNode for super | builtin-class super | builtin-class type | 35 | @@ -405,8 +400,6 @@ | g_class_init.py:50 | ControlFlowNode for self | self | class E | 46 | | g_class_init.py:52 | ControlFlowNode for FunctionExpr | Function meth | builtin-class function | 52 | | g_class_init.py:52 | ControlFlowNode for meth | Function meth | builtin-class function | 52 | -| g_class_init.py:53 | ControlFlowNode for Attribute | 'v2' | builtin-class str | 42 | -| g_class_init.py:53 | ControlFlowNode for Attribute | 'v3' | builtin-class str | 43 | | g_class_init.py:53 | ControlFlowNode for Compare | bool False | builtin-class bool | 53 | | g_class_init.py:53 | ControlFlowNode for Compare | bool True | builtin-class bool | 53 | | g_class_init.py:53 | ControlFlowNode for V2 | 'v2' | builtin-class str | 42 | @@ -434,6 +427,7 @@ | h_classes.py:12 | ControlFlowNode for Tuple | Tuple | builtin-class tuple | 12 | | h_classes.py:12 | ControlFlowNode for object | builtin-class object | builtin-class type | 12 | | h_classes.py:12 | ControlFlowNode for type | builtin-class type | builtin-class type | 12 | +| h_classes.py:12 | ControlFlowNode for type() | type() | builtin-class type | 12 | | h_classes.py:14 | ControlFlowNode for FunctionExpr | Function k | builtin-class function | 14 | | h_classes.py:14 | ControlFlowNode for k | Function k | builtin-class function | 14 | | h_classes.py:15 | ControlFlowNode for C | class C | builtin-class type | 3 | @@ -444,11 +438,12 @@ | h_classes.py:16 | ControlFlowNode for type | builtin-class type | builtin-class type | 16 | | h_classes.py:16 | ControlFlowNode for type() | builtin-class module | builtin-class type | 16 | | h_classes.py:17 | ControlFlowNode for type | builtin-class type | builtin-class type | 17 | -| h_classes.py:17 | ControlFlowNode for type() | *UNKNOWN TYPE* | builtin-class type | 17 | +| h_classes.py:17 | ControlFlowNode for type() | *UNKNOWN TYPE* | *UNKNOWN TYPE* | 17 | | h_classes.py:18 | ControlFlowNode for Dict | Dict | builtin-class dict | 18 | | h_classes.py:18 | ControlFlowNode for Tuple | Tuple | builtin-class tuple | 18 | | h_classes.py:18 | ControlFlowNode for object | builtin-class object | builtin-class type | 18 | | h_classes.py:18 | ControlFlowNode for type | builtin-class type | builtin-class type | 18 | +| h_classes.py:18 | ControlFlowNode for type() | type() | builtin-class type | 18 | | h_classes.py:23 | ControlFlowNode for Base | class Base | builtin-class type | 23 | | h_classes.py:23 | ControlFlowNode for ClassExpr | class Base | builtin-class type | 23 | | h_classes.py:23 | ControlFlowNode for object | builtin-class object | builtin-class type | 23 | @@ -599,6 +594,7 @@ | k_getsetattr.py:14 | ControlFlowNode for self | self | class C | 12 | | k_getsetattr.py:14 | ControlFlowNode for setattr | Builtin-function setattr | builtin-class builtin_function_or_method | 14 | | k_getsetattr.py:14 | ControlFlowNode for setattr() | NoneType None | builtin-class NoneType | 14 | +| k_getsetattr.py:15 | ControlFlowNode for Attribute | Attribute | builtin-class method | 15 | | k_getsetattr.py:15 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 6 | | k_getsetattr.py:15 | ControlFlowNode for self | self | class C | 12 | | k_getsetattr.py:16 | ControlFlowNode for Str | 'a' | builtin-class str | 16 | @@ -641,6 +637,7 @@ | l_calls.py:3 | ControlFlowNode for FunctionExpr | Function foo | builtin-class function | 3 | | l_calls.py:3 | ControlFlowNode for List | List | builtin-class list | 3 | | l_calls.py:3 | ControlFlowNode for foo | Function foo | builtin-class function | 3 | +| l_calls.py:4 | ControlFlowNode for Attribute | Attribute | builtin-class method | 4 | | l_calls.py:4 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 4 | | l_calls.py:4 | ControlFlowNode for Str | 'x' | builtin-class str | 4 | | l_calls.py:4 | ControlFlowNode for x | List | builtin-class list | 3 | @@ -661,7 +658,7 @@ | l_calls.py:14 | ControlFlowNode for classmethod() | classmethod() | builtin-class classmethod | 14 | | l_calls.py:15 | ControlFlowNode for FunctionExpr | Function cm | builtin-class function | 15 | | l_calls.py:15 | ControlFlowNode for cm | classmethod() | builtin-class classmethod | 14 | -| l_calls.py:16 | ControlFlowNode for cls | class Owner | builtin-class type | 23 | +| l_calls.py:16 | ControlFlowNode for cls | class Owner | builtin-class type | 12 | | l_calls.py:18 | ControlFlowNode for classmethod | builtin-class classmethod | builtin-class type | 18 | | l_calls.py:18 | ControlFlowNode for classmethod() | classmethod() | builtin-class classmethod | 18 | | l_calls.py:19 | ControlFlowNode for FunctionExpr | Function cm2 | builtin-class function | 19 | @@ -669,13 +666,15 @@ | l_calls.py:20 | ControlFlowNode for arg | int 1 | builtin-class int | 25 | | l_calls.py:23 | ControlFlowNode for FunctionExpr | Function m | builtin-class function | 23 | | l_calls.py:23 | ControlFlowNode for m | Function m | builtin-class function | 23 | -| l_calls.py:24 | ControlFlowNode for Attribute() | class Owner | builtin-class type | 23 | +| l_calls.py:24 | ControlFlowNode for Attribute | Attribute | builtin-class method | 24 | +| l_calls.py:24 | ControlFlowNode for Attribute() | class Owner | builtin-class type | 12 | | l_calls.py:24 | ControlFlowNode for IntegerLiteral | int 0 | builtin-class int | 24 | -| l_calls.py:24 | ControlFlowNode for a | class Owner | builtin-class type | 23 | +| l_calls.py:24 | ControlFlowNode for a | class Owner | builtin-class type | 12 | | l_calls.py:24 | ControlFlowNode for self | self | class Owner | 23 | +| l_calls.py:25 | ControlFlowNode for Attribute | Attribute | builtin-class method | 25 | | l_calls.py:25 | ControlFlowNode for Attribute() | int 1 | builtin-class int | 25 | | l_calls.py:25 | ControlFlowNode for IntegerLiteral | int 1 | builtin-class int | 25 | -| l_calls.py:25 | ControlFlowNode for a | class Owner | builtin-class type | 23 | +| l_calls.py:25 | ControlFlowNode for a | class Owner | builtin-class type | 12 | | s_scopes.py:4 | ControlFlowNode for True | bool True | builtin-class bool | 4 | | s_scopes.py:4 | ControlFlowNode for float | bool True | builtin-class bool | 4 | | s_scopes.py:7 | ControlFlowNode for C2 | class C2 | builtin-class type | 7 | diff --git a/python/ql/test/library-tests/PointsTo/new/Values.expected b/python/ql/test/library-tests/PointsTo/new/Values.expected index 0283e21fb5a8..c74d14d8db7d 100644 --- a/python/ql/test/library-tests/PointsTo/new/Values.expected +++ b/python/ql/test/library-tests/PointsTo/new/Values.expected @@ -370,12 +370,15 @@ | k_getsetattr.py:9 | ControlFlowNode for Str | runtime | 'a' | builtin-class str | | k_getsetattr.py:9 | ControlFlowNode for getattr | code/k_getsetattr.py:15 from runtime | Builtin-function getattr | builtin-class builtin_function_or_method | | k_getsetattr.py:9 | ControlFlowNode for getattr | runtime | Builtin-function getattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:9 | ControlFlowNode for getattr() | code/k_getsetattr.py:15 from runtime | int 0 | builtin-class int | +| k_getsetattr.py:9 | ControlFlowNode for getattr() | runtime | int 0 | builtin-class int | | k_getsetattr.py:9 | ControlFlowNode for self | code/k_getsetattr.py:15 from runtime | self instance of C | class C | | k_getsetattr.py:9 | ControlFlowNode for self | runtime | self instance of C | class C | | k_getsetattr.py:10 | ControlFlowNode for Str | code/k_getsetattr.py:15 from runtime | 'c' | builtin-class str | | k_getsetattr.py:10 | ControlFlowNode for Str | runtime | 'c' | builtin-class str | | k_getsetattr.py:10 | ControlFlowNode for getattr | code/k_getsetattr.py:15 from runtime | Builtin-function getattr | builtin-class builtin_function_or_method | | k_getsetattr.py:10 | ControlFlowNode for getattr | runtime | Builtin-function getattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:10 | ControlFlowNode for getattr() | code/k_getsetattr.py:15 from runtime | int 2 | builtin-class int | | k_getsetattr.py:10 | ControlFlowNode for self | code/k_getsetattr.py:15 from runtime | self instance of C | class C | | k_getsetattr.py:10 | ControlFlowNode for self | runtime | self instance of C | class C | | k_getsetattr.py:12 | ControlFlowNode for FunctionExpr | import | Function C.meth2 | builtin-class function | @@ -394,12 +397,15 @@ | k_getsetattr.py:15 | ControlFlowNode for self | runtime | self instance of C | class C | | k_getsetattr.py:16 | ControlFlowNode for Str | runtime | 'a' | builtin-class str | | k_getsetattr.py:16 | ControlFlowNode for getattr | runtime | Builtin-function getattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:16 | ControlFlowNode for getattr() | runtime | int 0 | builtin-class int | | k_getsetattr.py:16 | ControlFlowNode for self | runtime | self instance of C | class C | | k_getsetattr.py:17 | ControlFlowNode for Str | runtime | 'b' | builtin-class str | | k_getsetattr.py:17 | ControlFlowNode for getattr | runtime | Builtin-function getattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:17 | ControlFlowNode for getattr() | runtime | int 1 | builtin-class int | | k_getsetattr.py:17 | ControlFlowNode for self | runtime | self instance of C | class C | | k_getsetattr.py:18 | ControlFlowNode for Str | runtime | 'c' | builtin-class str | | k_getsetattr.py:18 | ControlFlowNode for getattr | runtime | Builtin-function getattr | builtin-class builtin_function_or_method | +| k_getsetattr.py:18 | ControlFlowNode for getattr() | runtime | int 2 | builtin-class int | | k_getsetattr.py:18 | ControlFlowNode for self | runtime | self instance of C | class C | | k_getsetattr.py:21 | ControlFlowNode for FunctionExpr | import | Function k | builtin-class function | | k_getsetattr.py:22 | ControlFlowNode for C | runtime | class C | builtin-class type | From 315fe0ab946d2daef9980a027d29868a895ddb69 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 10 May 2019 17:00:03 +0100 Subject: [PATCH 086/128] Python points-to: Update an expected file. --- .../test/library-tests/PointsTo/comparisons/PointsTo.expected | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected b/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected index edad5dbbda8b..29b17d92360b 100644 --- a/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected +++ b/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected @@ -1,3 +1,4 @@ +| 2 | bool False | | 2 | bool True | | 3 | bool False | | 6 | bool True | @@ -19,7 +20,6 @@ | 29 | bool False | | 30 | bool True | | 33 | bool False | -| 33 | bool True | | 34 | bool True | | 35 | bool False | | 36 | bool False | From 7244b9fc6fce806e1e9ecc8425f776ec4de878b5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 May 2019 09:58:30 +0100 Subject: [PATCH 087/128] Python points-to: Update some more expected test results. --- .../PointsTo/decorators/Test.expected | 1 - .../PointsTo/general/GlobalPointsTo.expected | 5 +++++ .../PointsTo/general/LocalPointsTo.expected | 10 ++++++++-- .../PointsTo/general/LocalPointsToType.expected | 10 ++++++++-- .../PointsTo/guarded/PointsTo.expected | 14 +++++++++----- .../PointsTo/guarded/PointsToWithType.expected | 14 +++++++++----- .../PointsTo/inheritance/MetaClass.expected | 3 --- 7 files changed, 39 insertions(+), 18 deletions(-) diff --git a/python/ql/test/library-tests/PointsTo/decorators/Test.expected b/python/ql/test/library-tests/PointsTo/decorators/Test.expected index 136adce143ff..06068e5206ed 100644 --- a/python/ql/test/library-tests/PointsTo/decorators/Test.expected +++ b/python/ql/test/library-tests/PointsTo/decorators/Test.expected @@ -1,3 +1,2 @@ | 41 | ControlFlowNode for func1 | Function func1 | test.py:23 | | 42 | ControlFlowNode for func2 | Function wrapper | test.py:10 | -| 43 | ControlFlowNode for func3 | Function wrapper | test.py:17 | diff --git a/python/ql/test/library-tests/PointsTo/general/GlobalPointsTo.expected b/python/ql/test/library-tests/PointsTo/general/GlobalPointsTo.expected index 627c29be92ba..e394f72918fa 100644 --- a/python/ql/test/library-tests/PointsTo/general/GlobalPointsTo.expected +++ b/python/ql/test/library-tests/PointsTo/general/GlobalPointsTo.expected @@ -32,7 +32,9 @@ | Class X | 36 | ControlFlowNode for classmethod() | classmethod() | | Class X | 37 | ControlFlowNode for FunctionExpr | Function method1 | | Class X | 37 | ControlFlowNode for method1 | classmethod() | +| Class X | 40 | ControlFlowNode for deco() | Function method2 | | Class X | 41 | ControlFlowNode for FunctionExpr | Function method2 | +| Class X | 41 | ControlFlowNode for method2 | Function method2 | | Module pointsto_test | 17 | ControlFlowNode for Attribute | list object | | Module pointsto_test | 17 | ControlFlowNode for Compare | bool False | | Module pointsto_test | 17 | ControlFlowNode for Compare | bool True | @@ -81,7 +83,9 @@ | Module pointsto_test | 64 | ControlFlowNode for dict | int 7 | | Module pointsto_test | 65 | ControlFlowNode for tuple | builtin-class tuple | | Module pointsto_test | 66 | ControlFlowNode for tuple | builtin-class tuple | +| Module pointsto_test | 69 | ControlFlowNode for Attribute | Attribute | | Module pointsto_test | 69 | ControlFlowNode for X | class X | +| Module pointsto_test | 70 | ControlFlowNode for Attribute | Function method2 | | Module pointsto_test | 70 | ControlFlowNode for X | class X | | Module pointsto_test | 72 | ControlFlowNode for ImportExpr | Module abc | | Module pointsto_test | 72 | ControlFlowNode for ImportMember | Function abstractmethod | @@ -99,6 +103,7 @@ | Module pointsto_test | 79 | ControlFlowNode for Tuple | Tuple | | Module pointsto_test | 79 | ControlFlowNode for object | builtin-class object | | Module pointsto_test | 79 | ControlFlowNode for type | builtin-class type | +| Module pointsto_test | 79 | ControlFlowNode for type() | type() | | Module pointsto_test | 81 | ControlFlowNode for FunctionExpr | Function k | | Module pointsto_test | 81 | ControlFlowNode for k | Function k | | Module pointsto_test | 88 | ControlFlowNode for FunctionExpr | Function outer | diff --git a/python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected b/python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected index f051a10e4a78..3dbab0c3f102 100644 --- a/python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected +++ b/python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected @@ -49,7 +49,9 @@ | 36 | ControlFlowNode for classmethod() | classmethod() | | 37 | ControlFlowNode for FunctionExpr | Function method1 | | 37 | ControlFlowNode for method1 | classmethod() | +| 40 | ControlFlowNode for deco() | Function method2 | | 41 | ControlFlowNode for FunctionExpr | Function method2 | +| 41 | ControlFlowNode for method2 | Function method2 | | 44 | ControlFlowNode for FunctionExpr | Function deco | | 44 | ControlFlowNode for deco | Function deco | | 47 | ControlFlowNode for v1 | class C | @@ -89,7 +91,9 @@ | 64 | ControlFlowNode for dict | int 7 | | 65 | ControlFlowNode for tuple | builtin-class tuple | | 66 | ControlFlowNode for tuple | builtin-class tuple | +| 69 | ControlFlowNode for Attribute | Attribute | | 69 | ControlFlowNode for X | class X | +| 70 | ControlFlowNode for Attribute | Function method2 | | 70 | ControlFlowNode for X | class X | | 72 | ControlFlowNode for ImportExpr | Module abc | | 72 | ControlFlowNode for ImportMember | Function abstractmethod | @@ -107,6 +111,7 @@ | 79 | ControlFlowNode for Tuple | Tuple | | 79 | ControlFlowNode for object | builtin-class object | | 79 | ControlFlowNode for type | builtin-class type | +| 79 | ControlFlowNode for type() | type() | | 81 | ControlFlowNode for FunctionExpr | Function k | | 81 | ControlFlowNode for k | Function k | | 82 | ControlFlowNode for C | class C | @@ -121,6 +126,7 @@ | 85 | ControlFlowNode for Tuple | Tuple | | 85 | ControlFlowNode for object | builtin-class object | | 85 | ControlFlowNode for type | builtin-class type | +| 85 | ControlFlowNode for type() | type() | | 88 | ControlFlowNode for FunctionExpr | Function outer | | 88 | ControlFlowNode for outer | Function outer | | 89 | ControlFlowNode for IntegerLiteral | int 1 | @@ -247,7 +253,7 @@ | 180 | ControlFlowNode for Derived4 | class Derived4 | | 182 | ControlFlowNode for FunctionExpr | Function __init__ | | 182 | ControlFlowNode for __init__ | Function __init__ | -| 183 | ControlFlowNode for Attribute | super().x | +| 183 | ControlFlowNode for Attribute | int 1 | | 183 | ControlFlowNode for Derived4 | class Derived4 | | 183 | ControlFlowNode for self | self | | 183 | ControlFlowNode for super | builtin-class super | @@ -335,6 +341,7 @@ | 263 | ControlFlowNode for Attribute | int 3 | | 263 | ControlFlowNode for IntegerLiteral | int 3 | | 263 | ControlFlowNode for self | self | +| 264 | ControlFlowNode for Attribute | int 0 | | 264 | ControlFlowNode for Attribute | int 3 | | 264 | ControlFlowNode for self | self | | 267 | ControlFlowNode for Derived4 | class Derived4 | @@ -349,7 +356,6 @@ | 273 | ControlFlowNode for self | self | | 275 | ControlFlowNode for FunctionExpr | Function add_node | | 275 | ControlFlowNode for add_node | Function add_node | -| 276 | ControlFlowNode for Attribute | Dict | | 276 | ControlFlowNode for IntegerLiteral | int 0 | | 276 | ControlFlowNode for Subscript | int 0 | | 276 | ControlFlowNode for self | self | diff --git a/python/ql/test/library-tests/PointsTo/general/LocalPointsToType.expected b/python/ql/test/library-tests/PointsTo/general/LocalPointsToType.expected index cdecda74f38b..09168c46448d 100644 --- a/python/ql/test/library-tests/PointsTo/general/LocalPointsToType.expected +++ b/python/ql/test/library-tests/PointsTo/general/LocalPointsToType.expected @@ -51,7 +51,9 @@ | 36 | ControlFlowNode for classmethod() | classmethod() | builtin-class classmethod | | 37 | ControlFlowNode for FunctionExpr | Function method1 | builtin-class function | | 37 | ControlFlowNode for method1 | classmethod() | builtin-class classmethod | +| 40 | ControlFlowNode for deco() | Function method2 | builtin-class function | | 41 | ControlFlowNode for FunctionExpr | Function method2 | builtin-class function | +| 41 | ControlFlowNode for method2 | Function method2 | builtin-class function | | 44 | ControlFlowNode for FunctionExpr | Function deco | builtin-class function | | 44 | ControlFlowNode for deco | Function deco | builtin-class function | | 47 | ControlFlowNode for v1 | class C | builtin-class type | @@ -92,7 +94,9 @@ | 64 | ControlFlowNode for dict | int 7 | builtin-class int | | 65 | ControlFlowNode for tuple | builtin-class tuple | builtin-class type | | 66 | ControlFlowNode for tuple | builtin-class tuple | builtin-class type | +| 69 | ControlFlowNode for Attribute | Attribute | builtin-class method | | 69 | ControlFlowNode for X | class X | builtin-class type | +| 70 | ControlFlowNode for Attribute | Function method2 | builtin-class function | | 70 | ControlFlowNode for X | class X | builtin-class type | | 72 | ControlFlowNode for ImportExpr | Module abc | builtin-class module | | 72 | ControlFlowNode for ImportMember | Function abstractmethod | builtin-class function | @@ -110,6 +114,7 @@ | 79 | ControlFlowNode for Tuple | Tuple | builtin-class tuple | | 79 | ControlFlowNode for object | builtin-class object | builtin-class type | | 79 | ControlFlowNode for type | builtin-class type | builtin-class type | +| 79 | ControlFlowNode for type() | type() | builtin-class type | | 81 | ControlFlowNode for FunctionExpr | Function k | builtin-class function | | 81 | ControlFlowNode for k | Function k | builtin-class function | | 82 | ControlFlowNode for C | class C | builtin-class type | @@ -124,6 +129,7 @@ | 85 | ControlFlowNode for Tuple | Tuple | builtin-class tuple | | 85 | ControlFlowNode for object | builtin-class object | builtin-class type | | 85 | ControlFlowNode for type | builtin-class type | builtin-class type | +| 85 | ControlFlowNode for type() | type() | builtin-class type | | 88 | ControlFlowNode for FunctionExpr | Function outer | builtin-class function | | 88 | ControlFlowNode for outer | Function outer | builtin-class function | | 89 | ControlFlowNode for IntegerLiteral | int 1 | builtin-class int | @@ -250,7 +256,7 @@ | 180 | ControlFlowNode for Derived4 | class Derived4 | builtin-class type | | 182 | ControlFlowNode for FunctionExpr | Function __init__ | builtin-class function | | 182 | ControlFlowNode for __init__ | Function __init__ | builtin-class function | -| 183 | ControlFlowNode for Attribute | super().x | builtin-class method | +| 183 | ControlFlowNode for Attribute | int 1 | builtin-class int | | 183 | ControlFlowNode for Derived4 | class Derived4 | builtin-class type | | 183 | ControlFlowNode for self | self | class Derived4 | | 183 | ControlFlowNode for super | builtin-class super | builtin-class type | @@ -338,6 +344,7 @@ | 263 | ControlFlowNode for Attribute | int 3 | builtin-class int | | 263 | ControlFlowNode for IntegerLiteral | int 3 | builtin-class int | | 263 | ControlFlowNode for self | self | class G | +| 264 | ControlFlowNode for Attribute | int 0 | builtin-class int | | 264 | ControlFlowNode for Attribute | int 3 | builtin-class int | | 264 | ControlFlowNode for self | self | class G | | 267 | ControlFlowNode for Derived4 | class Derived4 | builtin-class type | @@ -352,7 +359,6 @@ | 273 | ControlFlowNode for self | self | class DiGraph | | 275 | ControlFlowNode for FunctionExpr | Function add_node | builtin-class function | | 275 | ControlFlowNode for add_node | Function add_node | builtin-class function | -| 276 | ControlFlowNode for Attribute | Dict | builtin-class dict | | 276 | ControlFlowNode for IntegerLiteral | int 0 | builtin-class int | | 276 | ControlFlowNode for Subscript | int 0 | builtin-class int | | 276 | ControlFlowNode for self | self | class DiGraph | diff --git a/python/ql/test/library-tests/PointsTo/guarded/PointsTo.expected b/python/ql/test/library-tests/PointsTo/guarded/PointsTo.expected index 69d04f161247..3c9b6220e834 100644 --- a/python/ql/test/library-tests/PointsTo/guarded/PointsTo.expected +++ b/python/ql/test/library-tests/PointsTo/guarded/PointsTo.expected @@ -43,24 +43,28 @@ | test.py | 215 | ControlFlowNode for s | class C2 | 202 | | test.py | 220 | ControlFlowNode for s | int 0 | 219 | | test.py | 234 | ControlFlowNode for f | int 0 | 233 | -| test.py | 252 | ControlFlowNode for Attribute | int 1 | 242 | -| test.py | 254 | ControlFlowNode for Attribute | int 2 | 245 | | test.py | 272 | ControlFlowNode for x | int 1 | 271 | -| test.py | 276 | ControlFlowNode for Attribute | int 1 | 271 | | test.py | 286 | ControlFlowNode for y | NoneType None | 281 | | test.py | 297 | ControlFlowNode for y | NoneType None | 291 | | test.py | 301 | ControlFlowNode for x | NoneType None | 291 | | test.py | 308 | ControlFlowNode for z | int 7 | 305 | | test.py | 314 | ControlFlowNode for b | NoneType None | 311 | +| test.py | 330 | ControlFlowNode for Attribute | NoneType None | 324 | +| test.py | 330 | ControlFlowNode for Attribute | int 3 | 324 | +| test.py | 332 | ControlFlowNode for Attribute | NoneType None | 322 | | test.py | 332 | ControlFlowNode for Attribute | int 4 | 322 | -| test.py | 339 | ControlFlowNode for Attribute | int 4 | 322 | +| test.py | 337 | ControlFlowNode for Attribute | NoneType None | 324 | +| test.py | 337 | ControlFlowNode for Attribute | int 3 | 324 | +| test.py | 345 | ControlFlowNode for Attribute | NoneType None | 324 | +| test.py | 345 | ControlFlowNode for Attribute | int 3 | 324 | +| test.py | 347 | ControlFlowNode for Attribute | NoneType None | 322 | | test.py | 347 | ControlFlowNode for Attribute | int 4 | 322 | +| test.py | 357 | ControlFlowNode for g1 | float 7.0 | 356 | | test.py | 369 | ControlFlowNode for g2 | float 2.0 | 366 | | test.py | 382 | ControlFlowNode for g3 | bool True | 379 | | test.py | 389 | ControlFlowNode for g4 | int 7 | 396 | | test.py | 408 | ControlFlowNode for x | int 1 | 404 | | test.py | 420 | ControlFlowNode for Attribute | NoneType None | 418 | -| test.py | 427 | ControlFlowNode for Attribute | NoneType None | 418 | | test.py | 435 | ControlFlowNode for y | int 1 | 433 | | type_test.py | 5 | ControlFlowNode for d | Dict | 2 | | type_test.py | 14 | ControlFlowNode for x | int 0 | 11 | diff --git a/python/ql/test/library-tests/PointsTo/guarded/PointsToWithType.expected b/python/ql/test/library-tests/PointsTo/guarded/PointsToWithType.expected index 542f7ca864ca..d532fec3a9ba 100644 --- a/python/ql/test/library-tests/PointsTo/guarded/PointsToWithType.expected +++ b/python/ql/test/library-tests/PointsTo/guarded/PointsToWithType.expected @@ -43,24 +43,28 @@ | test.py | 215 | ControlFlowNode for s | class C2 | builtin-class type | 202 | | test.py | 220 | ControlFlowNode for s | int 0 | builtin-class int | 219 | | test.py | 234 | ControlFlowNode for f | int 0 | builtin-class int | 233 | -| test.py | 252 | ControlFlowNode for Attribute | int 1 | builtin-class int | 242 | -| test.py | 254 | ControlFlowNode for Attribute | int 2 | builtin-class int | 245 | | test.py | 272 | ControlFlowNode for x | int 1 | builtin-class int | 271 | -| test.py | 276 | ControlFlowNode for Attribute | int 1 | builtin-class int | 271 | | test.py | 286 | ControlFlowNode for y | NoneType None | builtin-class NoneType | 281 | | test.py | 297 | ControlFlowNode for y | NoneType None | builtin-class NoneType | 291 | | test.py | 301 | ControlFlowNode for x | NoneType None | builtin-class NoneType | 291 | | test.py | 308 | ControlFlowNode for z | int 7 | builtin-class int | 305 | | test.py | 314 | ControlFlowNode for b | NoneType None | builtin-class NoneType | 311 | +| test.py | 330 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 324 | +| test.py | 330 | ControlFlowNode for Attribute | int 3 | builtin-class int | 324 | +| test.py | 332 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 322 | | test.py | 332 | ControlFlowNode for Attribute | int 4 | builtin-class int | 322 | -| test.py | 339 | ControlFlowNode for Attribute | int 4 | builtin-class int | 322 | +| test.py | 337 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 324 | +| test.py | 337 | ControlFlowNode for Attribute | int 3 | builtin-class int | 324 | +| test.py | 345 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 324 | +| test.py | 345 | ControlFlowNode for Attribute | int 3 | builtin-class int | 324 | +| test.py | 347 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 322 | | test.py | 347 | ControlFlowNode for Attribute | int 4 | builtin-class int | 322 | +| test.py | 357 | ControlFlowNode for g1 | float 7.0 | builtin-class float | 356 | | test.py | 369 | ControlFlowNode for g2 | float 2.0 | builtin-class float | 366 | | test.py | 382 | ControlFlowNode for g3 | bool True | builtin-class bool | 379 | | test.py | 389 | ControlFlowNode for g4 | int 7 | builtin-class int | 396 | | test.py | 408 | ControlFlowNode for x | int 1 | builtin-class int | 404 | | test.py | 420 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 418 | -| test.py | 427 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 418 | | test.py | 435 | ControlFlowNode for y | int 1 | builtin-class int | 433 | | type_test.py | 5 | ControlFlowNode for d | Dict | builtin-class dict | 2 | | type_test.py | 14 | ControlFlowNode for x | int 0 | builtin-class int | 11 | diff --git a/python/ql/test/library-tests/PointsTo/inheritance/MetaClass.expected b/python/ql/test/library-tests/PointsTo/inheritance/MetaClass.expected index 6bdcc99296ef..0348b74c1fb4 100644 --- a/python/ql/test/library-tests/PointsTo/inheritance/MetaClass.expected +++ b/python/ql/test/library-tests/PointsTo/inheritance/MetaClass.expected @@ -5,8 +5,5 @@ | class Derived4 | builtin-class type | | class Derived5 | builtin-class type | | class Derived6 | builtin-class type | -| class Missing1 | builtin-class type | -| class Missing2 | builtin-class type | -| class Missing3 | builtin-class type | | class Wrong1 | builtin-class type | | class Wrong2 | builtin-class type | From beebd0e15ce180eb39790515ff7cf62a2b40496a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 May 2019 10:50:20 +0100 Subject: [PATCH 088/128] Python points-to: Be more specific when describing inheritance analysis failures. --- .../src/semmle/python/pointsto/PointsTo.qll | 27 ++++++++++++------- .../types/classes/FailedInference.expected | 21 --------------- 2 files changed, 17 insertions(+), 31 deletions(-) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index d1216272a21f..57621c9ba385 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1826,31 +1826,38 @@ cached module Types { /* Holds if type inference failed to compute the full class hierarchy for this class for the reason given. */ cached predicate failedInference(ClassObjectInternal cls, string reason) { - strictcount(cls.(PythonClassObjectInternal).getScope().getADecorator()) > 1 and reason = "Multiple decorators" + exists(int priority | + failedInference(cls, reason, priority) and + priority = max(int p | failedInference(cls, _, p)) + ) + } + + /* Holds if type inference failed to compute the full class hierarchy for this class for the reason given. */ + private predicate failedInference(ClassObjectInternal cls, string reason, int priority) { + strictcount(cls.(PythonClassObjectInternal).getScope().getADecorator()) > 1 and reason = "Multiple decorators" and priority = 0 or - exists(cls.(PythonClassObjectInternal).getScope().getADecorator()) and not six_add_metaclass(_, cls, _) and reason = "Decorator not understood" + exists(cls.(PythonClassObjectInternal).getScope().getADecorator()) and not six_add_metaclass(_, cls, _) and reason = "Decorator not understood" and priority = 1 or - reason = "Missing base " + missingBase(cls) + reason = "Missing base " + missingBase(cls) and priority = 6 or - not exists(ObjectInternal meta | meta = cls.getClass() and not meta = ObjectInternal::unknownClass()) and reason = "Failed to infer metaclass" + not exists(ObjectInternal meta | meta = cls.getClass() and not meta = ObjectInternal::unknownClass()) and reason = "Failed to infer metaclass" and priority = 4 or exists(int i, ObjectInternal base1, ObjectInternal base2 | base1 = getBase(cls, i) and base2 = getBase(cls, i) and base1 != base2 and reason = "Multiple bases at position " + i - ) + ) and priority = 6 or - duplicateBase(cls) and reason = "Duplicate bases classes" + duplicateBase(cls) and reason = "Duplicate bases classes" and priority = 6 or - not exists(getMro(cls)) and reason = "Failed to compute MRO" and not exists(missingBase(cls)) and not duplicateBase(cls) + not exists(getMro(cls)) and reason = "Failed to compute MRO" and priority = 3 or - exists(int i | failedInference(getBase(cls, i), _) and reason = "Failed inference for base class at position " + i) + exists(int i | failedInference(getBase(cls, i), _, _) and reason = "Failed inference for base class at position " + i) and priority = 5 } private int missingBase(ClassObjectInternal cls) { - exists(cls.(PythonClassObjectInternal).getScope().getBase(result)) - and + exists(cls.(PythonClassObjectInternal).getScope().getBase(result)) and not exists(ObjectInternal base | base = getBase(cls, result) and not base = ObjectInternal::unknownClass()) } diff --git a/python/ql/test/library-tests/types/classes/FailedInference.expected b/python/ql/test/library-tests/types/classes/FailedInference.expected index a8103a8847e1..ef8e8c9e224e 100644 --- a/python/ql/test/library-tests/types/classes/FailedInference.expected +++ b/python/ql/test/library-tests/types/classes/FailedInference.expected @@ -1,35 +1,14 @@ | class A | Missing base 0 | | class B | Failed inference for base class at position 0 | -| class B | Failed to compute MRO | | class C | Failed inference for base class at position 0 | -| class C | Failed to compute MRO | | class D | Duplicate bases classes | | class E | Duplicate bases classes | -| class E | Failed inference for base class at position 0 | -| class E | Failed inference for base class at position 1 | -| class E | Failed inference for base class at position 2 | -| class E | Failed inference for base class at position 3 | -| class E | Failed inference for base class at position 4 | -| class E | Failed inference for base class at position 5 | -| class E | Failed inference for base class at position 6 | -| class E | Failed inference for base class at position 7 | -| class E | Failed inference for base class at position 8 | -| class E | Failed inference for base class at position 9 | -| class E | Failed inference for base class at position 10 | | class G | Failed inference for base class at position 0 | -| class G | Failed to compute MRO | | class H | Failed inference for base class at position 0 | -| class H | Failed to compute MRO | | class J | Missing base 0 | | class L | Failed inference for base class at position 0 | -| class L | Failed to compute MRO | | class M | Failed inference for base class at position 0 | -| class M | Failed to compute MRO | | class N | Failed inference for base class at position 0 | -| class N | Failed to compute MRO | | class R | Failed inference for base class at position 0 | -| class R | Failed to compute MRO | | class S | Failed inference for base class at position 0 | -| class S | Failed to compute MRO | | class T | Failed inference for base class at position 0 | -| class T | Failed to compute MRO | From d74c76510f6f4b1414f4911733729b060c2fd0ee Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 May 2019 11:24:50 +0100 Subject: [PATCH 089/128] Python points-to: Don't track non-descriptor class attributes on instances. Update more tests. --- python/ql/src/semmle/python/objects/Instances.qll | 7 ++++++- .../PointsTo/attributes/Test.expected | 4 ++++ .../PointsTo/attributes/TestWithType.expected | 4 ++++ .../3/library-tests/PointsTo/attributes/test.py | 4 ++-- .../library-tests/classes/attr/list_attr.expected | 14 +++----------- python/ql/test/3/library-tests/six/test.expected | 15 +++++++++++++++ .../PointsTo/guarded/PointsTo.expected | 4 ---- .../PointsTo/guarded/PointsToWithType.expected | 4 ---- .../PointsTo/metaclass/Failed.expected | 1 - 9 files changed, 34 insertions(+), 23 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 2b93fb03e089..03c83ef6c68e 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -82,7 +82,12 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { exists(ObjectInternal cls_attr, CfgOrigin attr_orig | this.getClass().(ClassObjectInternal).lookup(name, cls_attr, attr_orig) | - cls_attr.isDescriptor() = false and value = cls_attr and origin = attr_orig + /* If class attribute is not a descriptor, that usually means it is some sort of + * default value and likely overridden by an instance attribute. In that case + * use `unknown` to signal that an attribute exists but to avoid false positives + * for due to using the default value. + */ + cls_attr.isDescriptor() = false and value = ObjectInternal::unknown() and origin = CfgOrigin::unknown() or cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this, value, origin) ) diff --git a/python/ql/test/3/library-tests/PointsTo/attributes/Test.expected b/python/ql/test/3/library-tests/PointsTo/attributes/Test.expected index c72603b26d1b..0adb7c86bef4 100644 --- a/python/ql/test/3/library-tests/PointsTo/attributes/Test.expected +++ b/python/ql/test/3/library-tests/PointsTo/attributes/Test.expected @@ -77,6 +77,7 @@ | 56 | ControlFlowNode for Attribute | int 20 | 56 | | 56 | ControlFlowNode for IntegerLiteral | int 20 | 56 | | 56 | ControlFlowNode for c2 | C() | 52 | +| 57 | ControlFlowNode for Attribute | int 1 | 10 | | 57 | ControlFlowNode for Attribute | int 10 | 54 | | 57 | ControlFlowNode for c1 | C() | 51 | | 58 | ControlFlowNode for Attribute | int 1 | 10 | @@ -114,6 +115,7 @@ | 72 | ControlFlowNode for Attribute | int 2 | 72 | | 72 | ControlFlowNode for IntegerLiteral | int 2 | 72 | | 72 | ControlFlowNode for self | self | 70 | +| 73 | ControlFlowNode for Attribute | Attribute | 73 | | 73 | ControlFlowNode for Attribute() | NoneType None | 64 | | 73 | ControlFlowNode for self | self | 70 | | 74 | ControlFlowNode for Attribute | int 0 | 65 | @@ -174,6 +176,7 @@ | 100 | ControlFlowNode for self | self | 98 | | 100 | ControlFlowNode for setattr | Builtin-function setattr | 100 | | 100 | ControlFlowNode for setattr() | NoneType None | 100 | +| 101 | ControlFlowNode for Attribute | Attribute | 101 | | 101 | ControlFlowNode for Attribute() | NoneType None | 92 | | 101 | ControlFlowNode for self | self | 98 | | 102 | ControlFlowNode for Str | str u'a' | 102 | @@ -198,5 +201,6 @@ | 109 | ControlFlowNode for self | self | 108 | | 109 | ControlFlowNode for setattr | Builtin-function setattr | 109 | | 109 | ControlFlowNode for setattr() | NoneType None | 109 | +| 111 | ControlFlowNode for Attribute | int 0 | 109 | | 111 | ControlFlowNode for G | class G | 106 | | 111 | ControlFlowNode for G() | G() | 111 | diff --git a/python/ql/test/3/library-tests/PointsTo/attributes/TestWithType.expected b/python/ql/test/3/library-tests/PointsTo/attributes/TestWithType.expected index cfc2c5f67477..92bd08344a03 100644 --- a/python/ql/test/3/library-tests/PointsTo/attributes/TestWithType.expected +++ b/python/ql/test/3/library-tests/PointsTo/attributes/TestWithType.expected @@ -77,6 +77,7 @@ | 56 | ControlFlowNode for Attribute | int 20 | builtin-class int | 56 | | 56 | ControlFlowNode for IntegerLiteral | int 20 | builtin-class int | 56 | | 56 | ControlFlowNode for c2 | C() | class C | 52 | +| 57 | ControlFlowNode for Attribute | int 1 | builtin-class int | 10 | | 57 | ControlFlowNode for Attribute | int 10 | builtin-class int | 54 | | 57 | ControlFlowNode for c1 | C() | class C | 51 | | 58 | ControlFlowNode for Attribute | int 1 | builtin-class int | 10 | @@ -114,6 +115,7 @@ | 72 | ControlFlowNode for Attribute | int 2 | builtin-class int | 72 | | 72 | ControlFlowNode for IntegerLiteral | int 2 | builtin-class int | 72 | | 72 | ControlFlowNode for self | self | class D | 70 | +| 73 | ControlFlowNode for Attribute | Attribute | builtin-class method | 73 | | 73 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 64 | | 73 | ControlFlowNode for self | self | class D | 70 | | 74 | ControlFlowNode for Attribute | int 0 | builtin-class int | 65 | @@ -174,6 +176,7 @@ | 100 | ControlFlowNode for self | self | class F | 98 | | 100 | ControlFlowNode for setattr | Builtin-function setattr | builtin-class builtin_function_or_method | 100 | | 100 | ControlFlowNode for setattr() | NoneType None | builtin-class NoneType | 100 | +| 101 | ControlFlowNode for Attribute | Attribute | builtin-class method | 101 | | 101 | ControlFlowNode for Attribute() | NoneType None | builtin-class NoneType | 92 | | 101 | ControlFlowNode for self | self | class F | 98 | | 102 | ControlFlowNode for Str | str u'a' | builtin-class str | 102 | @@ -198,5 +201,6 @@ | 109 | ControlFlowNode for self | self | class G | 108 | | 109 | ControlFlowNode for setattr | Builtin-function setattr | builtin-class builtin_function_or_method | 109 | | 109 | ControlFlowNode for setattr() | NoneType None | builtin-class NoneType | 109 | +| 111 | ControlFlowNode for Attribute | int 0 | builtin-class int | 109 | | 111 | ControlFlowNode for G | class G | builtin-class type | 106 | | 111 | ControlFlowNode for G() | G() | class G | 111 | diff --git a/python/ql/test/3/library-tests/PointsTo/attributes/test.py b/python/ql/test/3/library-tests/PointsTo/attributes/test.py index 749dfd3f0baf..0396a6e16066 100644 --- a/python/ql/test/3/library-tests/PointsTo/attributes/test.py +++ b/python/ql/test/3/library-tests/PointsTo/attributes/test.py @@ -54,7 +54,7 @@ def k(cond): c1.z = 10 if cond: c2.z = 20 - c1.z + c1.z # FP here due to self.attribute and local attribute c2.z c3.z c3.z = 30 @@ -85,7 +85,7 @@ def __init__(self, cond): E().x -#Make sure that we handle getattr and setattr as well as they are needed for protobuf stubs. +#Make sure that we handle getattr and setattr as well class F(object): diff --git a/python/ql/test/3/library-tests/classes/attr/list_attr.expected b/python/ql/test/3/library-tests/classes/attr/list_attr.expected index f9538876ae0f..075ded3309f1 100644 --- a/python/ql/test/3/library-tests/classes/attr/list_attr.expected +++ b/python/ql/test/3/library-tests/classes/attr/list_attr.expected @@ -1,12 +1,8 @@ | builtin-class list | __add__ | Builtin-method __add__ | -| builtin-class list | __class__ | Property __class__ | | builtin-class list | __contains__ | Builtin-method __contains__ | -| builtin-class list | __delattr__ | Builtin-method __delattr__ | | builtin-class list | __delitem__ | Builtin-method __delitem__ | -| builtin-class list | __dir__ | Builtin-method __dir__ | | builtin-class list | __doc__ | str u'list() -> new empty list\nlist(iterable) -> new list initialized from iterable's items' | | builtin-class list | __eq__ | Builtin-method __eq__ | -| builtin-class list | __format__ | Builtin-method __format__ | | builtin-class list | __ge__ | Builtin-method __ge__ | | builtin-class list | __getattribute__ | Builtin-method __getattribute__ | | builtin-class list | __getitem__ | Builtin-method __getitem__ | @@ -21,17 +17,12 @@ | builtin-class list | __lt__ | Builtin-method __lt__ | | builtin-class list | __mul__ | Builtin-method __mul__ | | builtin-class list | __ne__ | Builtin-method __ne__ | -| builtin-class list | __new__ | Builtin-method __new__ | -| builtin-class list | __reduce__ | Builtin-method __reduce__ | -| builtin-class list | __reduce_ex__ | Builtin-method __reduce_ex__ | +| builtin-class list | __new__ | builtin_function_or_method __new__ | | builtin-class list | __repr__ | Builtin-method __repr__ | | builtin-class list | __reversed__ | Builtin-method __reversed__ | | builtin-class list | __rmul__ | Builtin-method __rmul__ | -| builtin-class list | __setattr__ | Builtin-method __setattr__ | | builtin-class list | __setitem__ | Builtin-method __setitem__ | | builtin-class list | __sizeof__ | Builtin-method __sizeof__ | -| builtin-class list | __str__ | Builtin-method __str__ | -| builtin-class list | __subclasshook__ | classmethod_descriptor __subclasshook__ | | builtin-class list | append | Builtin-method append | | builtin-class list | clear | Builtin-method clear | | builtin-class list | copy | Builtin-method copy | @@ -60,13 +51,14 @@ | class DerivedFromBuiltin | __iadd__ | Builtin-method __iadd__ | | class DerivedFromBuiltin | __imul__ | Builtin-method __imul__ | | class DerivedFromBuiltin | __init__ | Builtin-method __init__ | +| class DerivedFromBuiltin | __init_subclass__ | classmethod_descriptor __init_subclass__ | | class DerivedFromBuiltin | __iter__ | Builtin-method __iter__ | | class DerivedFromBuiltin | __le__ | Builtin-method __le__ | | class DerivedFromBuiltin | __len__ | Builtin-method __len__ | | class DerivedFromBuiltin | __lt__ | Builtin-method __lt__ | | class DerivedFromBuiltin | __mul__ | Builtin-method __mul__ | | class DerivedFromBuiltin | __ne__ | Builtin-method __ne__ | -| class DerivedFromBuiltin | __new__ | Builtin-method __new__ | +| class DerivedFromBuiltin | __new__ | builtin_function_or_method __new__ | | class DerivedFromBuiltin | __reduce__ | Builtin-method __reduce__ | | class DerivedFromBuiltin | __reduce_ex__ | Builtin-method __reduce_ex__ | | class DerivedFromBuiltin | __repr__ | Builtin-method __repr__ | diff --git a/python/ql/test/3/library-tests/six/test.expected b/python/ql/test/3/library-tests/six/test.expected index 3ae0c6a5558f..2e52b0018409 100644 --- a/python/ql/test/3/library-tests/six/test.expected +++ b/python/ql/test/3/library-tests/six/test.expected @@ -28,6 +28,7 @@ | Module six | callable | Builtin-function callable | | Module six | callable | Function callable | | Module six | class_types | Tuple | +| Module six | create_bound_method | builtin-class method | | Module six | get_function_closure | Attribute() | | Module six | get_function_code | Attribute() | | Module six | get_function_defaults | Attribute() | @@ -45,8 +46,17 @@ | Module six | iterlists | Function iterlists | | Module six | itervalues | Function itervalues | | Module six | moves | Module six.moves | +| Module six | moves.__init__ | Module six.moves.__init__ | +| Module six | moves.urllib | Module six.moves.urllib | +| Module six | moves.urllib.__init__ | Module six.moves.urllib.__init__ | +| Module six | moves.urllib_error | Module six.moves.urllib_error | +| Module six | moves.urllib_parse | Module six.moves.urllib_parse | +| Module six | moves.urllib_request | Module six.moves.urllib_request | +| Module six | moves.urllib_response | Module six.moves.urllib_response | +| Module six | moves.urllib_robotparser | Module six.moves.urllib_robotparser | | Module six | next | Builtin-function next | | Module six | operator | Module operator | +| Module six | print_ | Function print_ | | Module six | remove_move | Function remove_move | | Module six | reraise | Function reraise | | Module six | string_types | Tuple | @@ -86,6 +96,7 @@ | Module six.__init__ | callable | Builtin-function callable | | Module six.__init__ | callable | Function callable | | Module six.__init__ | class_types | Tuple | +| Module six.__init__ | create_bound_method | builtin-class method | | Module six.__init__ | get_function_closure | Attribute() | | Module six.__init__ | get_function_code | Attribute() | | Module six.__init__ | get_function_defaults | Attribute() | @@ -105,6 +116,7 @@ | Module six.__init__ | moves | Module six.moves | | Module six.__init__ | next | Builtin-function next | | Module six.__init__ | operator | Module operator | +| Module six.__init__ | print_ | Function print_ | | Module six.__init__ | remove_move | Function remove_move | | Module six.__init__ | reraise | Function reraise | | Module six.__init__ | string_types | Tuple | @@ -120,6 +132,7 @@ | Module six.moves | PY3 | bool True | | Module six.moves | SimpleHTTPServer | Module http.server | | Module six.moves | StringIO | builtin-class _io.StringIO | +| Module six.moves | UserString | class UserString | | Module six.moves | __name__ | str u'six.moves' | | Module six.moves | _thread | Module _thread | | Module six.moves | builtins | Module builtins | @@ -161,6 +174,7 @@ | Module six.moves | tkinter_tksimpledialog | Module tkinter.simpledialog | | Module six.moves | tkinter_ttk | Module tkinter.ttk | | Module six.moves | urllib | Module six.moves.urllib | +| Module six.moves | urllib.__init__ | Module six.moves.urllib.__init__ | | Module six.moves | urllib_error | Module six.moves.urllib_error | | Module six.moves | urllib_parse | Module six.moves.urllib_parse | | Module six.moves | urllib_request | Module six.moves.urllib_request | @@ -176,6 +190,7 @@ | Module six.moves.__init__ | PY3 | bool True | | Module six.moves.__init__ | SimpleHTTPServer | Module http.server | | Module six.moves.__init__ | StringIO | builtin-class _io.StringIO | +| Module six.moves.__init__ | UserString | class UserString | | Module six.moves.__init__ | __name__ | str u'six.moves' | | Module six.moves.__init__ | _thread | Module _thread | | Module six.moves.__init__ | builtins | Module builtins | diff --git a/python/ql/test/library-tests/PointsTo/guarded/PointsTo.expected b/python/ql/test/library-tests/PointsTo/guarded/PointsTo.expected index 3c9b6220e834..61bb0bc19eb9 100644 --- a/python/ql/test/library-tests/PointsTo/guarded/PointsTo.expected +++ b/python/ql/test/library-tests/PointsTo/guarded/PointsTo.expected @@ -49,14 +49,10 @@ | test.py | 301 | ControlFlowNode for x | NoneType None | 291 | | test.py | 308 | ControlFlowNode for z | int 7 | 305 | | test.py | 314 | ControlFlowNode for b | NoneType None | 311 | -| test.py | 330 | ControlFlowNode for Attribute | NoneType None | 324 | -| test.py | 330 | ControlFlowNode for Attribute | int 3 | 324 | | test.py | 332 | ControlFlowNode for Attribute | NoneType None | 322 | | test.py | 332 | ControlFlowNode for Attribute | int 4 | 322 | | test.py | 337 | ControlFlowNode for Attribute | NoneType None | 324 | | test.py | 337 | ControlFlowNode for Attribute | int 3 | 324 | -| test.py | 345 | ControlFlowNode for Attribute | NoneType None | 324 | -| test.py | 345 | ControlFlowNode for Attribute | int 3 | 324 | | test.py | 347 | ControlFlowNode for Attribute | NoneType None | 322 | | test.py | 347 | ControlFlowNode for Attribute | int 4 | 322 | | test.py | 357 | ControlFlowNode for g1 | float 7.0 | 356 | diff --git a/python/ql/test/library-tests/PointsTo/guarded/PointsToWithType.expected b/python/ql/test/library-tests/PointsTo/guarded/PointsToWithType.expected index d532fec3a9ba..21d421b69bd5 100644 --- a/python/ql/test/library-tests/PointsTo/guarded/PointsToWithType.expected +++ b/python/ql/test/library-tests/PointsTo/guarded/PointsToWithType.expected @@ -49,14 +49,10 @@ | test.py | 301 | ControlFlowNode for x | NoneType None | builtin-class NoneType | 291 | | test.py | 308 | ControlFlowNode for z | int 7 | builtin-class int | 305 | | test.py | 314 | ControlFlowNode for b | NoneType None | builtin-class NoneType | 311 | -| test.py | 330 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 324 | -| test.py | 330 | ControlFlowNode for Attribute | int 3 | builtin-class int | 324 | | test.py | 332 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 322 | | test.py | 332 | ControlFlowNode for Attribute | int 4 | builtin-class int | 322 | | test.py | 337 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 324 | | test.py | 337 | ControlFlowNode for Attribute | int 3 | builtin-class int | 324 | -| test.py | 345 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 324 | -| test.py | 345 | ControlFlowNode for Attribute | int 3 | builtin-class int | 324 | | test.py | 347 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 322 | | test.py | 347 | ControlFlowNode for Attribute | int 4 | builtin-class int | 322 | | test.py | 357 | ControlFlowNode for g1 | float 7.0 | builtin-class float | 356 | diff --git a/python/ql/test/library-tests/PointsTo/metaclass/Failed.expected b/python/ql/test/library-tests/PointsTo/metaclass/Failed.expected index 8b560708beae..5a8e819e43ef 100644 --- a/python/ql/test/library-tests/PointsTo/metaclass/Failed.expected +++ b/python/ql/test/library-tests/PointsTo/metaclass/Failed.expected @@ -1,3 +1,2 @@ | test.py:26:1:26:17 | class C5 | Failed to infer metaclass | -| test.py:30:1:30:17 | class C6 | Decorator not understood | | test.py:30:1:30:17 | class C6 | Failed to infer metaclass | From 2905737a0e63d3c726e84d0b5dda275d48dec464 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 May 2019 12:00:38 +0100 Subject: [PATCH 090/128] Python points-to: Fix analysis of zero-arg super(). --- python/ql/src/semmle/python/objects/Instances.qll | 4 ++++ python/ql/src/semmle/python/objects/TObject.qll | 4 ++-- .../test/3/library-tests/PointsTo/inheritance/Calls.expected | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 03c83ef6c68e..168e70c2f30d 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -164,6 +164,10 @@ class SelfInstanceInternal extends TSelfInstance, ObjectInternal { this = TSelfInstance(_, _, result) } + ParameterDefinition getParameter() { + this = TSelfInstance(result, _, _) + } + override Builtin getBuiltin() { none() } diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 1e80dbc53d3b..a99401aba55a 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -220,8 +220,8 @@ private predicate super_noargs(CallNode instantiation, ObjectInternal self, Clas instantiation.getScope() = func and /* Implicit class argument is lexically enclosing scope */ func.getScope() = startclass.(PythonClassObjectInternal).getScope() and - /* Implicit 'self' is the 0th parameter */ - PointsToInternal::pointsTo(func.getArg(0).asName().getAFlowNode(), context, self, _) + /* Implicit 'self' is the `self` parameter of the enclosing function */ + self.(SelfInstanceInternal).getParameter().getParameter() = func.getArg(0) ) } diff --git a/python/ql/test/3/library-tests/PointsTo/inheritance/Calls.expected b/python/ql/test/3/library-tests/PointsTo/inheritance/Calls.expected index 58310ddb687e..06d56a628ad9 100644 --- a/python/ql/test/3/library-tests/PointsTo/inheritance/Calls.expected +++ b/python/ql/test/3/library-tests/PointsTo/inheritance/Calls.expected @@ -2,5 +2,4 @@ | 14 | Function meth | 8 | | 22 | Function meth | 13 | | 27 | Function meth | 8 | -| 27 | Function meth | 13 | | 32 | Function meth | 26 | From 0491fe13567727b1f3ffaadf18e072d97622f93a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 May 2019 12:11:19 +0100 Subject: [PATCH 091/128] Python points-to: Update 'analysis' query test results. --- python/ql/src/analysis/KeyPointsToFailure.ql | 5 +-- .../pointsto/CallGraphEfficiency.expected | 4 +-- .../CallGraphMarginalEfficiency.expected | 4 +-- .../pointsto/KeyPointsToFailure.expected | 33 ------------------- .../analysis/pointsto/Pruned.expected | 2 +- 5 files changed, 8 insertions(+), 40 deletions(-) diff --git a/python/ql/src/analysis/KeyPointsToFailure.ql b/python/ql/src/analysis/KeyPointsToFailure.ql index 79439134a070..90bc77aa0d78 100644 --- a/python/ql/src/analysis/KeyPointsToFailure.ql +++ b/python/ql/src/analysis/KeyPointsToFailure.ql @@ -8,11 +8,12 @@ */ import python +import semmle.python.pointsto.PointsTo predicate points_to_failure(Expr e) { - exists(ControlFlowNode f | + exists(ControlFlowNode f | f = e.getAFlowNode() | - not exists(f.pointsTo()) + not PointsTo::pointsTo(f, _, _, _) ) } diff --git a/python/ql/test/query-tests/analysis/pointsto/CallGraphEfficiency.expected b/python/ql/test/query-tests/analysis/pointsto/CallGraphEfficiency.expected index c558a04665f8..d72bc6028d6e 100644 --- a/python/ql/test/query-tests/analysis/pointsto/CallGraphEfficiency.expected +++ b/python/ql/test/query-tests/analysis/pointsto/CallGraphEfficiency.expected @@ -1,2 +1,2 @@ -| 0 | 31 | 31 | 100.0 | -| 1 | 4 | 40 | 10.0 | +| 0 | 53 | 53 | 100.0 | +| 1 | 3 | 39 | 7.6923076923076925 | diff --git a/python/ql/test/query-tests/analysis/pointsto/CallGraphMarginalEfficiency.expected b/python/ql/test/query-tests/analysis/pointsto/CallGraphMarginalEfficiency.expected index d22bc9bb4816..d72bc6028d6e 100644 --- a/python/ql/test/query-tests/analysis/pointsto/CallGraphMarginalEfficiency.expected +++ b/python/ql/test/query-tests/analysis/pointsto/CallGraphMarginalEfficiency.expected @@ -1,2 +1,2 @@ -| 0 | 31 | 31 | 100.0 | -| 1 | 3 | 40 | 7.5 | +| 0 | 53 | 53 | 100.0 | +| 1 | 3 | 39 | 7.6923076923076925 | diff --git a/python/ql/test/query-tests/analysis/pointsto/KeyPointsToFailure.expected b/python/ql/test/query-tests/analysis/pointsto/KeyPointsToFailure.expected index 197ffae7992a..e69de29bb2d1 100644 --- a/python/ql/test/query-tests/analysis/pointsto/KeyPointsToFailure.expected +++ b/python/ql/test/query-tests/analysis/pointsto/KeyPointsToFailure.expected @@ -1,33 +0,0 @@ -| test.py:30:13:30:21 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:37:21:37:29 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:43:9:43:17 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:50:17:50:25 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:52:13:52:21 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:55:13:55:21 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:65:43:65:51 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:118:43:118:55 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:129:24:129:32 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:133:24:133:32 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:137:24:137:32 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:141:24:141:32 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:155:16:155:30 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:229:17:229:27 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:230:18:230:31 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:422:16:422:29 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:443:21:443:34 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:445:18:445:31 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:454:21:454:34 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:460:20:460:33 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:461:18:461:31 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:470:20:470:33 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:476:20:476:33 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:477:20:477:33 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:509:16:509:33 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:531:21:531:29 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:645:35:645:48 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:656:39:656:52 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:663:20:663:28 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:676:23:676:33 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:686:30:686:53 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:712:37:712:49 | Attribute | Expression does not 'point-to' any object, but all its sources do. | -| test.py:712:55:712:63 | Attribute | Expression does not 'point-to' any object, but all its sources do. | diff --git a/python/ql/test/query-tests/analysis/pointsto/Pruned.expected b/python/ql/test/query-tests/analysis/pointsto/Pruned.expected index 53854800e6f2..54879e0059e9 100644 --- a/python/ql/test/query-tests/analysis/pointsto/Pruned.expected +++ b/python/ql/test/query-tests/analysis/pointsto/Pruned.expected @@ -1 +1 @@ -| 1418 | +| 1284 | From 26044f20c78bf9e691ce14657c332aded5f4a88d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 May 2019 12:59:19 +0100 Subject: [PATCH 092/128] Python points-to: Fix up tuple inequality analysis. --- python/ql/src/semmle/python/pointsto/PointsTo.qll | 4 ++-- .../test/2/library-tests/PointsTo/import_time/Pruned.ql | 2 +- .../PointsTo/import_time/VersionGuard.expected | 3 --- .../2/library-tests/PointsTo/import_time/VersionGuard.ql | 8 -------- 4 files changed, 3 insertions(+), 14 deletions(-) delete mode 100644 python/ql/test/2/library-tests/PointsTo/import_time/VersionGuard.expected delete mode 100644 python/ql/test/2/library-tests/PointsTo/import_time/VersionGuard.ql diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 57621c9ba385..95d373d92a75 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1343,9 +1343,9 @@ module Expressions { or n = other.length() and n = val.length() and result = 0 or - result != 0 and result = compare_unbound(val.getItem(n), val.getItem(n)) + result != 0 and result = compare_unbound(val.getItem(n), other.getItem(n)) or - compare_unbound(val.getItem(n), val.getItem(n)) = 0 and result = compare_sequence(val, other, n+1) + compare_unbound(val.getItem(n), other.getItem(n)) = 0 and result = compare_sequence(val, other, n+1) ) } diff --git a/python/ql/test/2/library-tests/PointsTo/import_time/Pruned.ql b/python/ql/test/2/library-tests/PointsTo/import_time/Pruned.ql index bafb54346f24..2cfdb6407cf8 100644 --- a/python/ql/test/2/library-tests/PointsTo/import_time/Pruned.ql +++ b/python/ql/test/2/library-tests/PointsTo/import_time/Pruned.ql @@ -5,7 +5,7 @@ import semmle.python.pointsto.PointsToContext from ControlFlowNode f, Location l, Context c -where not PointsTo::Test::reachableBlock(f.getBasicBlock(), c) and c.isImport() and +where not PointsToInternal::reachableBlock(f.getBasicBlock(), c) and c.isImport() and (f.getNode() instanceof FunctionExpr or f.getNode() instanceof ClassExpr) and l = f.getLocation() and l.getFile().getName().matches("%test.py") select l.getStartLine() \ No newline at end of file diff --git a/python/ql/test/2/library-tests/PointsTo/import_time/VersionGuard.expected b/python/ql/test/2/library-tests/PointsTo/import_time/VersionGuard.expected deleted file mode 100644 index 8f48841c0983..000000000000 --- a/python/ql/test/2/library-tests/PointsTo/import_time/VersionGuard.expected +++ /dev/null @@ -1,3 +0,0 @@ -| module.py | 6 | BasicBlock | true | -| test.py | 27 | BasicBlock | true | -| test.py | 41 | BasicBlock | true | diff --git a/python/ql/test/2/library-tests/PointsTo/import_time/VersionGuard.ql b/python/ql/test/2/library-tests/PointsTo/import_time/VersionGuard.ql deleted file mode 100644 index 5a8cc496340b..000000000000 --- a/python/ql/test/2/library-tests/PointsTo/import_time/VersionGuard.ql +++ /dev/null @@ -1,8 +0,0 @@ - -import python -import semmle.python.types.Version - -from VersionGuard vg, Location l, boolean b -where l = vg.getLastNode().getLocation() -and (if vg.isTrue() then b = true else b = false) -select l.getFile().getName(), l.getStartLine(), vg.toString(), b \ No newline at end of file From 1f00c3b2488825b96cc0d49349950c1dfb3f87a2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 May 2019 14:20:43 +0100 Subject: [PATCH 093/128] Python points-to: Fix up metaclass determination for Python 2. --- .../ql/src/semmle/python/objects/ObjectInternal.qll | 5 +++++ python/ql/src/semmle/python/pointsto/PointsTo.qll | 12 ++++++++---- python/ql/test/2/library-tests/classes/mro/C3.ql | 12 +++++++----- python/ql/test/2/library-tests/classes/mro/mro.ql | 6 +----- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 4b477dffcef0..0765d769f8e4 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -426,6 +426,11 @@ module ObjectInternal { result = TBuiltinClassObject(Builtin::special("super")) } + /** The old-style class type (Python 2 only) */ + ObjectInternal classType() { + result = TBuiltinClassObject(Builtin::special("ClassType")) + } + } /** Helper for boolean predicates returning both `true` and `false` */ diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 95d373d92a75..54b80ad3a317 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1692,9 +1692,9 @@ cached module Types { } cached ClassObjectInternal getMetaClass(PythonClassObjectInternal cls) { - result = declaredMetaClass(cls) - or - hasDeclaredMetaclass(cls) = false and result = getInheritedMetaclass(cls) + result = declaredMetaClass(cls) + or + hasDeclaredMetaclass(cls) = false and result = getInheritedMetaclass(cls) } private ClassObjectInternal declaredMetaClass(PythonClassObjectInternal cls) { @@ -1799,7 +1799,9 @@ cached module Types { c = cls.(PythonClassObjectInternal).getScope() and n = count(c.getABase()) | - result = ObjectInternal::builtin("type") + result = ObjectInternal::type() and major_version() = 3 + or + result = ObjectInternal::classType() and major_version() = 2 ) or exists(ClassObjectInternal meta1, ClassObjectInternal meta2 | @@ -1811,6 +1813,8 @@ cached module Types { or improperSuperType(meta2) = meta1 and result = meta2 or + meta2 = ObjectInternal::classType() and result = meta1 + or /* Make sure we have a metaclass, even if base is unknown */ meta1 = ObjectInternal::unknownClass() and result = ObjectInternal::builtin("type") or diff --git a/python/ql/test/2/library-tests/classes/mro/C3.ql b/python/ql/test/2/library-tests/classes/mro/C3.ql index 19d158a9606c..d04f49af51c5 100644 --- a/python/ql/test/2/library-tests/classes/mro/C3.ql +++ b/python/ql/test/2/library-tests/classes/mro/C3.ql @@ -1,15 +1,17 @@ import python import semmle.python.pointsto.MRO +import semmle.python.pointsto.PointsTo +import semmle.python.objects.ObjectInternal -ClassList mro(ClassObject cls) { - if cls.isNewStyle() then - result = new_style_mro(cls) +ClassList mro(ClassObjectInternal cls) { + if Types::isNewStyle(cls) then + result = Mro::newStyleMro(cls) else - result = old_style_mro(cls) + result = Mro::oldStyleMro(cls) } -from ClassObject cls +from ClassObjectInternal cls where not cls.isBuiltin() select cls.toString(), mro(cls) diff --git a/python/ql/test/2/library-tests/classes/mro/mro.ql b/python/ql/test/2/library-tests/classes/mro/mro.ql index 87f7e35c7bcf..0695e6ce5b8c 100644 --- a/python/ql/test/2/library-tests/classes/mro/mro.ql +++ b/python/ql/test/2/library-tests/classes/mro/mro.ql @@ -1,8 +1,4 @@ -/** - * @name class_attr - * @kind test - * @problem.severity warning - */ + import python From 33043d6a8a4b90b00ab72f1095b59aa209853ffa Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 May 2019 14:35:22 +0100 Subject: [PATCH 094/128] Python: update exptected test results. --- .../2/library-tests/classes/attr/list_attr.expected | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/python/ql/test/2/library-tests/classes/attr/list_attr.expected b/python/ql/test/2/library-tests/classes/attr/list_attr.expected index 1c07fe874944..b281b8f9e9a6 100644 --- a/python/ql/test/2/library-tests/classes/attr/list_attr.expected +++ b/python/ql/test/2/library-tests/classes/attr/list_attr.expected @@ -1,12 +1,9 @@ | builtin-class list | __add__ | Builtin-method __add__ | -| builtin-class list | __class__ | Property __class__ | | builtin-class list | __contains__ | Builtin-method __contains__ | -| builtin-class list | __delattr__ | Builtin-method __delattr__ | | builtin-class list | __delitem__ | Builtin-method __delitem__ | | builtin-class list | __delslice__ | Builtin-method __delslice__ | | builtin-class list | __doc__ | str b'list() -> new empty list\nlist(iterable) -> new list initialized from iterable's items' | | builtin-class list | __eq__ | Builtin-method __eq__ | -| builtin-class list | __format__ | Builtin-method __format__ | | builtin-class list | __ge__ | Builtin-method __ge__ | | builtin-class list | __getattribute__ | Builtin-method __getattribute__ | | builtin-class list | __getitem__ | Builtin-method __getitem__ | @@ -22,18 +19,13 @@ | builtin-class list | __lt__ | Builtin-method __lt__ | | builtin-class list | __mul__ | Builtin-method __mul__ | | builtin-class list | __ne__ | Builtin-method __ne__ | -| builtin-class list | __new__ | Builtin-method __new__ | -| builtin-class list | __reduce__ | Builtin-method __reduce__ | -| builtin-class list | __reduce_ex__ | Builtin-method __reduce_ex__ | +| builtin-class list | __new__ | builtin_function_or_method __new__ | | builtin-class list | __repr__ | Builtin-method __repr__ | | builtin-class list | __reversed__ | Builtin-method __reversed__ | | builtin-class list | __rmul__ | Builtin-method __rmul__ | -| builtin-class list | __setattr__ | Builtin-method __setattr__ | | builtin-class list | __setitem__ | Builtin-method __setitem__ | | builtin-class list | __setslice__ | Builtin-method __setslice__ | | builtin-class list | __sizeof__ | Builtin-method __sizeof__ | -| builtin-class list | __str__ | Builtin-method __str__ | -| builtin-class list | __subclasshook__ | classmethod_descriptor __subclasshook__ | | builtin-class list | append | Builtin-method append | | builtin-class list | count | Builtin-method count | | builtin-class list | extend | Builtin-method extend | @@ -67,7 +59,7 @@ | class DerivedFromBuiltin | __lt__ | Builtin-method __lt__ | | class DerivedFromBuiltin | __mul__ | Builtin-method __mul__ | | class DerivedFromBuiltin | __ne__ | Builtin-method __ne__ | -| class DerivedFromBuiltin | __new__ | Builtin-method __new__ | +| class DerivedFromBuiltin | __new__ | builtin_function_or_method __new__ | | class DerivedFromBuiltin | __reduce__ | Builtin-method __reduce__ | | class DerivedFromBuiltin | __reduce_ex__ | Builtin-method __reduce_ex__ | | class DerivedFromBuiltin | __repr__ | Builtin-method __repr__ | From 5f9a807392c95802ef8ba2c90947818e9a03a071 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 May 2019 15:08:58 +0100 Subject: [PATCH 095/128] Python points-to: Fix handling of builtin-method objects. --- .../ql/src/semmle/python/objects/Constants.qll | 15 +++++++++++++-- .../types/classes/new_style.expected | 16 +--------------- .../2/library-tests/types/classes/new_style.ql | 1 + .../2/library-tests/types/functions/odasa6418.py | 2 ++ 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index d01495102c3f..48163dd050e9 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -38,7 +38,11 @@ abstract class ConstantObjectInternal extends ObjectInternal { } pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - none() + PointsToInternal::attributeRequired(this, name) and + exists(ObjectInternal cls_attr, CfgOrigin attr_orig | + this.getClass().(ClassObjectInternal).lookup(name, cls_attr, attr_orig) and + cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this, value, origin) + ) } pragma [noinline] override predicate attributesUnknown() { none() } @@ -51,7 +55,14 @@ abstract class ConstantObjectInternal extends ObjectInternal { pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } - pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { + exists(ClassObjectInternal cls | + receiver_type(_, name, this, cls) and + cls.lookup(name, descriptor, _) and + descriptor.isDescriptor() = true + ) and + this = instance + } } diff --git a/python/ql/test/2/library-tests/types/classes/new_style.expected b/python/ql/test/2/library-tests/types/classes/new_style.expected index 662c4ec91c30..b7457e73e18e 100644 --- a/python/ql/test/2/library-tests/types/classes/new_style.expected +++ b/python/ql/test/2/library-tests/types/classes/new_style.expected @@ -1,24 +1,10 @@ -| class A | old | -| class B | old | -| class C | old | -| class D | new | -| class E | new | -| class G | new | -| class H | new | -| class J | new | -| class L | old | -| class M | old | | class MyDict | new | | class MyList | new | -| class N | old | | class O1 | old | | class O2 | old | | class O3 | old | | class O4 | old | -| class R | new | -| class S | new | -| class T | new | | class W | new | | class X | new | | class Y | new | -| class Z | new | \ No newline at end of file +| class Z | new | diff --git a/python/ql/test/2/library-tests/types/classes/new_style.ql b/python/ql/test/2/library-tests/types/classes/new_style.ql index e25ec44b2336..5c66eff3e603 100644 --- a/python/ql/test/2/library-tests/types/classes/new_style.ql +++ b/python/ql/test/2/library-tests/types/classes/new_style.ql @@ -3,6 +3,7 @@ import python from ClassObject cls, string style where not cls.isC() and +not cls.failedInference() and ( if cls.isNewStyle() then style = "new" diff --git a/python/ql/test/2/library-tests/types/functions/odasa6418.py b/python/ql/test/2/library-tests/types/functions/odasa6418.py index 958433c4287b..2665791aeddc 100644 --- a/python/ql/test/2/library-tests/types/functions/odasa6418.py +++ b/python/ql/test/2/library-tests/types/functions/odasa6418.py @@ -17,3 +17,5 @@ def fail(message): # To get the FP result reported in ODASA-6418, #bump_version must be called (directly or transitively) from the module scope bump_version() +str.join +"".join From 972ac0fdbd12bdc103bfbc3171d62e3cdb938569 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 May 2019 15:15:27 +0100 Subject: [PATCH 096/128] Python: Update test result. --- .../ql/test/2/library-tests/six/test.expected | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/python/ql/test/2/library-tests/six/test.expected b/python/ql/test/2/library-tests/six/test.expected index 82d1c43b06e1..8462f2b2464f 100644 --- a/python/ql/test/2/library-tests/six/test.expected +++ b/python/ql/test/2/library-tests/six/test.expected @@ -1,6 +1,5 @@ | Module six | BytesIO | class StringIO | | Module six | Iterator | class Iterator | -| Module six | MAXSIZE | int() | | Module six | PY2 | bool True | | Module six | PY3 | bool False | | Module six | StringIO | class StringIO | @@ -46,8 +45,17 @@ | Module six | iterlists | Function iterlists | | Module six | itervalues | Function itervalues | | Module six | moves | Module six.moves | +| Module six | moves.__init__ | Module six.moves.__init__ | +| Module six | moves.urllib | Module six.moves.urllib | +| Module six | moves.urllib.__init__ | Module six.moves.urllib.__init__ | +| Module six | moves.urllib_error | Module six.moves.urllib_error | +| Module six | moves.urllib_parse | Module six.moves.urllib_parse | +| Module six | moves.urllib_request | Module six.moves.urllib_request | +| Module six | moves.urllib_response | Module six.moves.urllib_response | +| Module six | moves.urllib_robotparser | Module six.moves.urllib_robotparser | | Module six | next | Builtin-function next | | Module six | operator | Module operator | +| Module six | print_ | Function print_ | | Module six | remove_move | Function remove_move | | Module six | reraise | Function reraise | | Module six | string_types | Tuple | @@ -59,7 +67,6 @@ | Module six | with_metaclass | Function with_metaclass | | Module six.__init__ | BytesIO | class StringIO | | Module six.__init__ | Iterator | class Iterator | -| Module six.__init__ | MAXSIZE | int() | | Module six.__init__ | PY2 | bool True | | Module six.__init__ | PY3 | bool False | | Module six.__init__ | StringIO | class StringIO | @@ -107,6 +114,7 @@ | Module six.__init__ | moves | Module six.moves | | Module six.__init__ | next | Builtin-function next | | Module six.__init__ | operator | Module operator | +| Module six.__init__ | print_ | Function print_ | | Module six.__init__ | remove_move | Function remove_move | | Module six.__init__ | reraise | Function reraise | | Module six.__init__ | string_types | Tuple | @@ -123,6 +131,8 @@ | Module six.moves | SimpleHTTPServer | Module SimpleHTTPServer | | Module six.moves | StringIO | class StringIO | | Module six.moves | UserDict | class UserDict | +| Module six.moves | UserList | class UserList | +| Module six.moves | UserString | class UserString | | Module six.moves | __name__ | str b'six.moves' | | Module six.moves | _dummy_thread | Module dummy_thread | | Module six.moves | _thread | Module thread | @@ -164,6 +174,7 @@ | Module six.moves | tkinter_tksimpledialog | Module tkSimpleDialog | | Module six.moves | tkinter_ttk | Module ttk | | Module six.moves | urllib | Module six.moves.urllib | +| Module six.moves | urllib.__init__ | Module six.moves.urllib.__init__ | | Module six.moves | urllib_error | Module six.moves.urllib_error | | Module six.moves | urllib_parse | Module six.moves.urllib_parse | | Module six.moves | urllib_request | Module six.moves.urllib_request | @@ -181,6 +192,8 @@ | Module six.moves.__init__ | SimpleHTTPServer | Module SimpleHTTPServer | | Module six.moves.__init__ | StringIO | class StringIO | | Module six.moves.__init__ | UserDict | class UserDict | +| Module six.moves.__init__ | UserList | class UserList | +| Module six.moves.__init__ | UserString | class UserString | | Module six.moves.__init__ | __name__ | str b'six.moves' | | Module six.moves.__init__ | _dummy_thread | Module dummy_thread | | Module six.moves.__init__ | _thread | Module thread | From a03e101e4f671b8ad340021e731a50898644a25d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 May 2019 17:15:01 +0100 Subject: [PATCH 097/128] Python points-to: Improve performance. --- python/ql/src/semmle/python/Flow.qll | 12 ++ .../src/semmle/python/pointsto/PointsTo.qll | 117 ++++++++---------- 2 files changed, 67 insertions(+), 62 deletions(-) diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll index 9b6cc156b927..04570fb3dc4b 100755 --- a/python/ql/src/semmle/python/Flow.qll +++ b/python/ql/src/semmle/python/Flow.qll @@ -1014,6 +1014,18 @@ class BasicBlock extends @py_flow_node { result = this.getLastNode().getAFalseSuccessor().getBasicBlock() } + /** Gets an exceptional successor to this basic block */ + BasicBlock getAnUnconditionalSuccessor() { + result = this.getASuccessor() and + not result = this.getATrueSuccessor() and + not result = this.getAFalseSuccessor() + } + + /** Gets an exceptional successor to this basic block */ + BasicBlock getAnExceptionalSuccessor() { + result = this.getLastNode().getAnExceptionalSuccessor().getBasicBlock() + } + /** Gets the scope of this block */ pragma [nomagic] Scope getScope() { exists(ControlFlowNode n | diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 54b80ad3a317..040cce33c8df 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -214,43 +214,28 @@ cached module PointsToInternal { /* Holds if BasicBlock `b` is reachable, given the context `context`. */ cached predicate reachableBlock(BasicBlock b, PointsToContext context) { - context.appliesToScope(b.getScope()) and not exists(ConditionBlock guard | guard.controls(b, _)) - or - exists(ConditionBlock guard | - guard = b.getImmediatelyControllingBlock() and - reachableBlock(guard, context) - | - allowsFlow(guard, b, context) - or - /* Assume the true edge of an assert is reachable (except for assert 0/False) */ - guard.controls(b, true) and - exists(Assert a, Expr test | - a.getTest() = test and - guard.getLastNode().getNode() = test and - not test instanceof ImmutableLiteral - ) - ) - } - - pragma [noopt] - private predicate allowsFlow(ConditionBlock guard, BasicBlock b, PointsToContext context) { - exists(ObjectInternal value, boolean sense, ControlFlowNode test | - test = guard.getLastNode() and - pointsTo(test, context, value, _) and - sense = value.booleanValue() and - guard.controls(b, sense) + exists(Scope scope | + context.appliesToScope(scope) and + scope.getEntryNode().getBasicBlock() = b ) + or + reachableEdge(_, b, context) } - /* Holds if the edge `pred` -> `succ` is reachable, given the context `context`. - */ - pragma [noopt] - cached predicate controlledReachableEdge(BasicBlock pred, BasicBlock succ, PointsToContext context) { - exists(ConditionBlock guard, ObjectInternal value, boolean sense, ControlFlowNode test | - test = guard.getLastNode() and - pointsTo(test, context, value, _) and - sense = value.booleanValue() and - guard.controlsEdge(pred, succ, sense) + private predicate reachableEdge(BasicBlock pred, BasicBlock succ, PointsToContext context) { + reachableBlock(pred, context) and + ( + pred.getAnUnconditionalSuccessor() = succ + or + exists(ObjectInternal value, boolean sense, ControlFlowNode test | + test = pred.getLastNode() and + pointsTo(test, context, value, _) and + sense = value.booleanValue() + | + sense = true and succ = pred.getATrueSuccessor() + or + sense = false and succ = pred.getAFalseSuccessor() + ) ) } @@ -519,13 +504,18 @@ cached module PointsToInternal { /** Holds if the phi-function `phi` refers to `(value, origin)` given the context `context`. */ pragma [nomagic] private predicate ssa_phi_points_to(PhiFunction phi, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - exists(EssaVariable input, BasicBlock pred | - input = phi.getInput(pred) and + exists(EssaVariable input | + ssa_phi_reachable_from_input(phi, context, input) and variablePointsTo(input, context, value, origin) - | - controlledReachableEdge(pred, phi.getBasicBlock(), context) - or - not exists(ConditionBlock guard | guard.controlsEdge(pred, phi.getBasicBlock(), _)) + ) + } + + /* Helper for ssa_phi_points_to */ + pragma [noinline] + private predicate ssa_phi_reachable_from_input(PhiFunction phi, PointsToContext context, EssaVariable input) { + exists(BasicBlock pred | + input = phi.getInput(pred) and + reachableEdge(pred, phi.getBasicBlock(), context) ) } @@ -839,9 +829,7 @@ module InterProceduralPointsTo { predicate parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { self_parameter_points_to(def, context, value, origin) or - positional_parameter_points_to(def, context, value, origin) - or - named_parameter_points_to(def, context, value, origin) + normal_parameter_points_to(def, context, value, origin) or default_parameter_points_to(def, context, value, origin) or @@ -850,7 +838,7 @@ module InterProceduralPointsTo { /** Helper for `parameter_points_to` */ pragma [noinline] - private predicate positional_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + private predicate normal_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(PointsToContext caller, ControlFlowNode arg | PointsToInternal::pointsTo(arg, caller, value, origin) and callsite_argument_transfer(arg, caller, def, context) @@ -882,16 +870,6 @@ module InterProceduralPointsTo { ) } - /** Helper for `parameter_points_to` */ - pragma [noinline] - private predicate named_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - exists(CallNode call, PointsToContext caller, PythonFunctionObjectInternal func, string name | - context.fromCall(call, func, caller) and - def.getParameter() = func.getScope().getArgByName(name) and - PointsToInternal::pointsTo(call.getArgByName(name), caller, value, origin) - ) - } - /** Helper for parameter_points_to */ pragma [noinline] private predicate default_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { @@ -939,10 +917,18 @@ module InterProceduralPointsTo { /** Holds if the `(argument, caller)` pair matches up with `(param, callee)` pair across call. */ cached predicate callsite_argument_transfer(ControlFlowNode argument, PointsToContext caller, ParameterDefinition param, PointsToContext callee) { - exists(CallNode call, Function func, int n, int offset | - callsite_calls_function(call, caller, func, callee, offset) and - argument = call.getArg(n) and - param.getParameter() = func.getArg(n+offset) + exists(CallNode call, Function func, int offset | + callsite_calls_function(call, caller, func, callee, offset) + | + exists(int n | + argument = call.getArg(n) and + param.getParameter() = func.getArg(n+offset) + ) + or + exists(string name | + argument = call.getArgByName(name) and + param.getParameter() = func.getArgByName(name) + ) ) } @@ -1334,6 +1320,7 @@ module Expressions { val.strValue() = other.strValue() and result = 1 } + pragma [nomagic] private int compare_sequence(SequenceObjectInternal val, SequenceObjectInternal other, int n) { inequalityTest(_, _, _, val, other, _, _) and ( @@ -1342,11 +1329,17 @@ module Expressions { n = other.length() and val.length() > n and result = 1 or n = other.length() and n = val.length() and result = 0 - or - result != 0 and result = compare_unbound(val.getItem(n), other.getItem(n)) - or - compare_unbound(val.getItem(n), other.getItem(n)) = 0 and result = compare_sequence(val, other, n+1) ) + or + result != 0 and result = compare_item(val, other, n) + or + compare_item(val, other, n) = 0 and result = compare_sequence(val, other, n+1) + } + + pragma [noinline] + private int compare_item(SequenceObjectInternal val, SequenceObjectInternal other, int n) { + inequalityTest(_, _, _, val, other, _, _) and + result = compare_unbound(val.getItem(n), other.getItem(n)) } pragma [noinline] From 1c20336eb982d64a0be3aefd946b105f33bd838a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 May 2019 17:24:02 +0100 Subject: [PATCH 098/128] Python points-to. Fix attribute lookup for type. --- python/ql/src/semmle/python/objects/Classes.qll | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index d5786ef30832..48a362665ad4 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -284,7 +284,8 @@ class TypeInternal extends ClassObjectInternal, TType { } override predicate lookup(string name, ObjectInternal value, CfgOrigin origin) { - none() + value = ObjectInternal::fromBuiltin(Builtin::special("type").getMember(name)) and + origin = CfgOrigin::unknown() } pragma [noinline] override predicate attributesUnknown() { any() } From a8dc2908de3890c9f3f4ba4d9d74b0d69b1a7cd2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 May 2019 17:29:24 +0100 Subject: [PATCH 099/128] Python: Update test results. --- python/ql/test/library-tests/web/falcon/Taint.expected | 4 ---- python/ql/test/query-tests/analysis/pointsto/Pruned.expected | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/python/ql/test/library-tests/web/falcon/Taint.expected b/python/ql/test/library-tests/web/falcon/Taint.expected index 8c00c5886bc3..e61ebf238709 100644 --- a/python/ql/test/library-tests/web/falcon/Taint.expected +++ b/python/ql/test/library-tests/web/falcon/Taint.expected @@ -3,16 +3,12 @@ | test.py:10 | Attribute | file[externally controlled string] | | test.py:10 | Attribute() | externally controlled string | | test.py:10 | req | falcon.request | -| test.py:11 | Attribute() | externally controlled string | | test.py:11 | Attribute() | json[externally controlled string] | | test.py:11 | raw_json | externally controlled string | | test.py:12 | resp | falcon.response | -| test.py:13 | Dict | {externally controlled string} | | test.py:13 | Dict | {json[externally controlled string]} | -| test.py:15 | result | externally controlled string | | test.py:15 | result | json[externally controlled string] | | test.py:17 | resp | falcon.response | -| test.py:17 | result | {externally controlled string} | | test.py:17 | result | {json[externally controlled string]} | | test.py:19 | req | falcon.request | | test.py:19 | resp | falcon.response | diff --git a/python/ql/test/query-tests/analysis/pointsto/Pruned.expected b/python/ql/test/query-tests/analysis/pointsto/Pruned.expected index 54879e0059e9..d770590ea65d 100644 --- a/python/ql/test/query-tests/analysis/pointsto/Pruned.expected +++ b/python/ql/test/query-tests/analysis/pointsto/Pruned.expected @@ -1 +1 @@ -| 1284 | +| 1304 | From a5ff527ac2a0e0bc47f0aadd82d4ebd671cb6403 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 May 2019 10:45:43 +0100 Subject: [PATCH 100/128] Python: Remove test due to instability across minor versions. --- .../classes/attr/list_attr.expected | 83 ------------------- .../3/library-tests/classes/attr/list_attr.ql | 16 ---- 2 files changed, 99 deletions(-) delete mode 100644 python/ql/test/3/library-tests/classes/attr/list_attr.expected delete mode 100644 python/ql/test/3/library-tests/classes/attr/list_attr.ql diff --git a/python/ql/test/3/library-tests/classes/attr/list_attr.expected b/python/ql/test/3/library-tests/classes/attr/list_attr.expected deleted file mode 100644 index 075ded3309f1..000000000000 --- a/python/ql/test/3/library-tests/classes/attr/list_attr.expected +++ /dev/null @@ -1,83 +0,0 @@ -| builtin-class list | __add__ | Builtin-method __add__ | -| builtin-class list | __contains__ | Builtin-method __contains__ | -| builtin-class list | __delitem__ | Builtin-method __delitem__ | -| builtin-class list | __doc__ | str u'list() -> new empty list\nlist(iterable) -> new list initialized from iterable's items' | -| builtin-class list | __eq__ | Builtin-method __eq__ | -| builtin-class list | __ge__ | Builtin-method __ge__ | -| builtin-class list | __getattribute__ | Builtin-method __getattribute__ | -| builtin-class list | __getitem__ | Builtin-method __getitem__ | -| builtin-class list | __gt__ | Builtin-method __gt__ | -| builtin-class list | __hash__ | NoneType None | -| builtin-class list | __iadd__ | Builtin-method __iadd__ | -| builtin-class list | __imul__ | Builtin-method __imul__ | -| builtin-class list | __init__ | Builtin-method __init__ | -| builtin-class list | __iter__ | Builtin-method __iter__ | -| builtin-class list | __le__ | Builtin-method __le__ | -| builtin-class list | __len__ | Builtin-method __len__ | -| builtin-class list | __lt__ | Builtin-method __lt__ | -| builtin-class list | __mul__ | Builtin-method __mul__ | -| builtin-class list | __ne__ | Builtin-method __ne__ | -| builtin-class list | __new__ | builtin_function_or_method __new__ | -| builtin-class list | __repr__ | Builtin-method __repr__ | -| builtin-class list | __reversed__ | Builtin-method __reversed__ | -| builtin-class list | __rmul__ | Builtin-method __rmul__ | -| builtin-class list | __setitem__ | Builtin-method __setitem__ | -| builtin-class list | __sizeof__ | Builtin-method __sizeof__ | -| builtin-class list | append | Builtin-method append | -| builtin-class list | clear | Builtin-method clear | -| builtin-class list | copy | Builtin-method copy | -| builtin-class list | count | Builtin-method count | -| builtin-class list | extend | Builtin-method extend | -| builtin-class list | index | Builtin-method index | -| builtin-class list | insert | Builtin-method insert | -| builtin-class list | pop | Builtin-method pop | -| builtin-class list | remove | Builtin-method remove | -| builtin-class list | reverse | Builtin-method reverse | -| builtin-class list | sort | Builtin-method sort | -| class DerivedFromBuiltin | __add__ | Builtin-method __add__ | -| class DerivedFromBuiltin | __class__ | Property __class__ | -| class DerivedFromBuiltin | __contains__ | Builtin-method __contains__ | -| class DerivedFromBuiltin | __delattr__ | Builtin-method __delattr__ | -| class DerivedFromBuiltin | __delitem__ | Builtin-method __delitem__ | -| class DerivedFromBuiltin | __dir__ | Builtin-method __dir__ | -| class DerivedFromBuiltin | __doc__ | str u'list() -> new empty list\nlist(iterable) -> new list initialized from iterable's items' | -| class DerivedFromBuiltin | __eq__ | Builtin-method __eq__ | -| class DerivedFromBuiltin | __format__ | Builtin-method __format__ | -| class DerivedFromBuiltin | __ge__ | Builtin-method __ge__ | -| class DerivedFromBuiltin | __getattribute__ | Builtin-method __getattribute__ | -| class DerivedFromBuiltin | __getitem__ | Builtin-method __getitem__ | -| class DerivedFromBuiltin | __gt__ | Builtin-method __gt__ | -| class DerivedFromBuiltin | __hash__ | NoneType None | -| class DerivedFromBuiltin | __iadd__ | Builtin-method __iadd__ | -| class DerivedFromBuiltin | __imul__ | Builtin-method __imul__ | -| class DerivedFromBuiltin | __init__ | Builtin-method __init__ | -| class DerivedFromBuiltin | __init_subclass__ | classmethod_descriptor __init_subclass__ | -| class DerivedFromBuiltin | __iter__ | Builtin-method __iter__ | -| class DerivedFromBuiltin | __le__ | Builtin-method __le__ | -| class DerivedFromBuiltin | __len__ | Builtin-method __len__ | -| class DerivedFromBuiltin | __lt__ | Builtin-method __lt__ | -| class DerivedFromBuiltin | __mul__ | Builtin-method __mul__ | -| class DerivedFromBuiltin | __ne__ | Builtin-method __ne__ | -| class DerivedFromBuiltin | __new__ | builtin_function_or_method __new__ | -| class DerivedFromBuiltin | __reduce__ | Builtin-method __reduce__ | -| class DerivedFromBuiltin | __reduce_ex__ | Builtin-method __reduce_ex__ | -| class DerivedFromBuiltin | __repr__ | Builtin-method __repr__ | -| class DerivedFromBuiltin | __reversed__ | Builtin-method __reversed__ | -| class DerivedFromBuiltin | __rmul__ | Builtin-method __rmul__ | -| class DerivedFromBuiltin | __setattr__ | Builtin-method __setattr__ | -| class DerivedFromBuiltin | __setitem__ | Builtin-method __setitem__ | -| class DerivedFromBuiltin | __sizeof__ | Builtin-method __sizeof__ | -| class DerivedFromBuiltin | __str__ | Builtin-method __str__ | -| class DerivedFromBuiltin | __subclasshook__ | classmethod_descriptor __subclasshook__ | -| class DerivedFromBuiltin | append | Builtin-method append | -| class DerivedFromBuiltin | clear | Builtin-method clear | -| class DerivedFromBuiltin | copy | Builtin-method copy | -| class DerivedFromBuiltin | count | Builtin-method count | -| class DerivedFromBuiltin | extend | Builtin-method extend | -| class DerivedFromBuiltin | index | Builtin-method index | -| class DerivedFromBuiltin | insert | Builtin-method insert | -| class DerivedFromBuiltin | meth5 | Function meth5 | -| class DerivedFromBuiltin | pop | Builtin-method pop | -| class DerivedFromBuiltin | remove | Builtin-method remove | -| class DerivedFromBuiltin | reverse | Builtin-method reverse | -| class DerivedFromBuiltin | sort | Builtin-method sort | diff --git a/python/ql/test/3/library-tests/classes/attr/list_attr.ql b/python/ql/test/3/library-tests/classes/attr/list_attr.ql deleted file mode 100644 index 2977030252a7..000000000000 --- a/python/ql/test/3/library-tests/classes/attr/list_attr.ql +++ /dev/null @@ -1,16 +0,0 @@ -/** - * @name class_attr - * @kind test - * @problem.severity warning - */ - -import python - -from ClassObject cls, string name, Object what -where -(cls.getName() = "list" or - cls.getASuperType().getName() = "list" -) -and -cls.lookupAttribute(name) = what -select cls.toString(), name, what.toString() From 0afcb11a1384ecdc1f163f516e26080f4f86d308 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 May 2019 11:50:21 +0100 Subject: [PATCH 101/128] Python points-to: Make sure reachability can skip over if-statements. --- python/ql/src/semmle/python/Flow.qll | 16 ++++++++++++++++ .../ql/src/semmle/python/pointsto/PointsTo.qll | 5 +++++ .../analysis/pointsto/Pruned.expected | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll index 04570fb3dc4b..7e07085666b1 100755 --- a/python/ql/src/semmle/python/Flow.qll +++ b/python/ql/src/semmle/python/Flow.qll @@ -1077,6 +1077,22 @@ class BasicBlock extends @py_flow_node { not result.(ConditionBlock).controls(this, _) } + /** Holds if flow from this BasicBlock always reaches `succ` + */ + predicate alwaysReaches(BasicBlock succ) { + succ = this + or + strictcount(this.getASuccessor()) = 1 + and succ = this.getASuccessor() + or + forex(BasicBlock immsucc | + immsucc = this.getASuccessor() + | + immsucc.alwaysReaches(succ) + ) + + } + } private predicate start_bb_likely_reachable(BasicBlock b) { diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 040cce33c8df..fea66f5050b1 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -220,6 +220,11 @@ cached module PointsToInternal { ) or reachableEdge(_, b, context) + or + exists(BasicBlock pred | + reachableBlock(pred, context) and + pred.alwaysReaches(b) + ) } private predicate reachableEdge(BasicBlock pred, BasicBlock succ, PointsToContext context) { diff --git a/python/ql/test/query-tests/analysis/pointsto/Pruned.expected b/python/ql/test/query-tests/analysis/pointsto/Pruned.expected index d770590ea65d..54879e0059e9 100644 --- a/python/ql/test/query-tests/analysis/pointsto/Pruned.expected +++ b/python/ql/test/query-tests/analysis/pointsto/Pruned.expected @@ -1 +1 @@ -| 1304 | +| 1284 | From 8f74f918c781693e074d46036400622863e29eff Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 May 2019 14:49:29 +0100 Subject: [PATCH 102/128] Python points-to: Speed up call-points-to. --- .../src/semmle/python/pointsto/PointsTo.qll | 72 ++++++++++--------- 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index fea66f5050b1..d153bdd01381 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -789,46 +789,48 @@ module InterProceduralPointsTo { } pragma [noinline] - private predicate call_points_to_simple(CallNode f, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + private predicate call_points_to_simple(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(ObjectInternal func | call(f, context, func) | - exists(PointsToContext callee | + exists(CfgOrigin orig, PointsToContext callee | callee.fromCall(f, context) and - func.callResult(callee, value, origin) + func.callResult(callee, value, orig) and + origin = orig.asCfgNodeOrHere(f) ) or context.untrackableCall(f) and - value = ObjectInternal::unknown() and origin = CfgOrigin::unknown() + value = ObjectInternal::unknown() and origin = f or - func.callResult(value, origin) and + exists(CfgOrigin orig | + func.callResult(value, orig) and + origin = orig.asCfgNodeOrHere(f) + ) and context.appliesTo(f) ) } pragma [noinline] predicate call_points_to(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - exists(ObjectInternal returnValue, CfgOrigin resultOrigin | - call_points_to_simple(f, context, returnValue, resultOrigin) - | - /* Either not a decorator, or we understand the return value */ - (returnValue != ObjectInternal::unknown() or not f.isDecoratorCall()) and - value = returnValue and origin = resultOrigin.asCfgNodeOrHere(f) - or - /* A decorator and we don't understand it. Use the original, undecorated value */ - f.isDecoratorCall() and returnValue = ObjectInternal::unknown() and - PointsToInternal::pointsTo(f.getArg(0), context, value, origin) - or - Types::six_add_metaclass(f, value, _) and - PointsToInternal::pointsTo(f.getArg(0), context, value, origin) - ) + /* Either not a decorator, or we understand the return value */ + (value != ObjectInternal::unknown() or not f.isDecoratorCall()) and + call_points_to_simple(f, context, value, origin) + or + call_result_is_first_argument(f, context) and + PointsToInternal::pointsTo(f.getArg(0), context, value, origin) or Expressions::typeCallPointsTo(f, context, value, origin, _, _) + } + + /** Helper for call_points_to to improve join-order */ + private predicate call_result_is_first_argument(CallNode f, PointsToContext context) { + Types::six_add_metaclass(f, context, _, _) or - Types::six_add_metaclass(f, value, _) and - PointsToInternal::pointsTo(f.getArg(0), context, value, origin) + /* A decorator and we don't understand it. Use the original, undecorated value */ + f.isDecoratorCall() and call_points_to_simple(f, context, ObjectInternal::unknown(), _) } + /** Points-to for parameter. `def foo(param): ...`. */ pragma [noinline] predicate parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { @@ -1327,13 +1329,14 @@ module Expressions { pragma [nomagic] private int compare_sequence(SequenceObjectInternal val, SequenceObjectInternal other, int n) { - inequalityTest(_, _, _, val, other, _, _) and - ( - n = val.length() and other.length() > n and result = -1 + exists(int vlen, int olen | + sequence_lengths_in_comparison(val, other, vlen, olen) + | + n = vlen and olen > n and result = -1 or - n = other.length() and val.length() > n and result = 1 + n = olen and vlen > n and result = 1 or - n = other.length() and n = val.length() and result = 0 + n = olen and n = vlen and result = 0 ) or result != 0 and result = compare_item(val, other, n) @@ -1341,6 +1344,11 @@ module Expressions { compare_item(val, other, n) = 0 and result = compare_sequence(val, other, n+1) } + private predicate sequence_lengths_in_comparison(SequenceObjectInternal val, SequenceObjectInternal other, int vlen, int olen) { + inequalityTest(_, _, _, val, other, _, _) and + vlen = val.length() and olen = other.length() + } + pragma [noinline] private int compare_item(SequenceObjectInternal val, SequenceObjectInternal other, int n) { inequalityTest(_, _, _, val, other, _, _) and @@ -1704,7 +1712,7 @@ cached module Types { ) or exists(ControlFlowNode meta | - six_add_metaclass(_, cls, meta) and + six_add_metaclass(_, _, cls, meta) and PointsToInternal::pointsTo(meta, _, result, _) ) } @@ -1758,16 +1766,16 @@ cached module Types { cls.(ImportTimeScope).entryEdge(result.getAUse(), _) } - cached predicate six_add_metaclass(CallNode decorator_call, ClassObjectInternal decorated, ControlFlowNode metaclass) { + cached predicate six_add_metaclass(CallNode decorator_call, PointsToContext context, ClassObjectInternal decorated, ControlFlowNode metaclass) { exists(CallNode decorator | - PointsToInternal::pointsTo(decorator_call.getArg(0), _, decorated, _) and + PointsToInternal::pointsTo(decorator_call.getArg(0), context, decorated, _) and decorator = decorator_call.getFunction() and decorator.getArg(0) = metaclass | - PointsToInternal::pointsTo(decorator.getFunction(), _, six_add_metaclass_function(), _) + PointsToInternal::pointsTo(decorator.getFunction(), context, six_add_metaclass_function(), _) or exists(ModuleObjectInternal six | six.getName() = "six" and - PointsToInternal::pointsTo(decorator.getFunction().(AttrNode).getObject("add_metaclass"), _, six, _) + PointsToInternal::pointsTo(decorator.getFunction().(AttrNode).getObject("add_metaclass"), context, six, _) ) ) } @@ -1838,7 +1846,7 @@ cached module Types { private predicate failedInference(ClassObjectInternal cls, string reason, int priority) { strictcount(cls.(PythonClassObjectInternal).getScope().getADecorator()) > 1 and reason = "Multiple decorators" and priority = 0 or - exists(cls.(PythonClassObjectInternal).getScope().getADecorator()) and not six_add_metaclass(_, cls, _) and reason = "Decorator not understood" and priority = 1 + exists(cls.(PythonClassObjectInternal).getScope().getADecorator()) and not six_add_metaclass(_, _, cls, _) and reason = "Decorator not understood" and priority = 1 or reason = "Missing base " + missingBase(cls) and priority = 6 or From b10a7cd3a474f0a9d780db6d85167b204371d3f2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 May 2019 14:50:09 +0100 Subject: [PATCH 103/128] Python points-to: Make behviour of instances more consistent. --- .../src/semmle/python/objects/Instances.qll | 97 +++++++++++-------- 1 file changed, 54 insertions(+), 43 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 168e70c2f30d..7f8d6498f817 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -8,7 +8,45 @@ private import semmle.python.pointsto.MRO private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins -class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { +abstract class InstanceObject extends ObjectInternal { + + pragma [nomagic] + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + PointsToInternal::attributeRequired(this, name) and + ( + exists(ObjectInternal cls_attr | + this.getClass().(ClassObjectInternal).lookup(name, cls_attr, _) + | + /* If class attribute is not a descriptor, that usually means it is some sort of + * default value and likely overridden by an instance attribute. In that case + * use `unknown` to signal that an attribute exists but to avoid false positives + * f using the default value. + */ + cls_attr.isDescriptor() = false and value = ObjectInternal::unknown() and origin = CfgOrigin::unknown() + or + cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this, value, origin) + ) + or + exists(EssaVariable self, PythonFunctionObjectInternal init, Context callee | + this.initializer(init, callee) and + self_variable_reaching_init_exit(self) and + self.getScope() = init.getScope() and + AttributePointsTo::variableAttributePointsTo(self, callee, name, value, origin) + ) + ) + } + + abstract predicate initializer(PythonFunctionObjectInternal init, Context callee); + +} + +private predicate self_variable_reaching_init_exit(EssaVariable self) { + BaseFlow::reaches_exit(self) and + self.getSourceVariable().(Variable).isSelf() and + self.getScope().getName() = "__init__" +} + +class SpecificInstanceInternal extends TSpecificInstance, InstanceObject { override string toString() { result = this.getOrigin().getNode().toString() @@ -76,35 +114,6 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { none() } - pragma [nomagic] - override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - PointsToInternal::attributeRequired(this, name) and - exists(ObjectInternal cls_attr, CfgOrigin attr_orig | - this.getClass().(ClassObjectInternal).lookup(name, cls_attr, attr_orig) - | - /* If class attribute is not a descriptor, that usually means it is some sort of - * default value and likely overridden by an instance attribute. In that case - * use `unknown` to signal that an attribute exists but to avoid false positives - * for due to using the default value. - */ - cls_attr.isDescriptor() = false and value = ObjectInternal::unknown() and origin = CfgOrigin::unknown() - or - cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this, value, origin) - ) - or - exists(EssaVariable self, PythonFunctionObjectInternal init, Context callee | - BaseFlow::reaches_exit(self) and - self.getSourceVariable().(Variable).isSelf() and - self.getScope() = init.getScope() and - exists(CallNode call, Context caller, ClassObjectInternal cls | - this = TSpecificInstance(call, cls, caller) and - callee.fromCall(this.getOrigin(), caller) and - cls.lookup("__init__", init, _) - ) and - AttributePointsTo::variableAttributePointsTo(self, callee, name, value, origin) - ) - } - pragma [noinline] override predicate attributesUnknown() { any() } override predicate subscriptUnknown() { any() } @@ -128,10 +137,18 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal { result = lengthFromClass(this.getClass()) } + override predicate initializer(PythonFunctionObjectInternal init, Context callee) { + exists(CallNode call, Context caller, ClassObjectInternal cls | + this = TSpecificInstance(call, cls, caller) and + callee.fromCall(this.getOrigin(), caller) and + cls.lookup("__init__", init, _) + ) + } + } -class SelfInstanceInternal extends TSelfInstance, ObjectInternal { +class SelfInstanceInternal extends TSelfInstance, InstanceObject { override string toString() { result = "self instance of " + this.getClass().(ClassObjectInternal).getName() @@ -201,17 +218,6 @@ class SelfInstanceInternal extends TSelfInstance, ObjectInternal { none() } - pragma [nomagic] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - PointsToInternal::attributeRequired(this, name) and - exists(ObjectInternal cls_attr, CfgOrigin attr_orig | - this.getClass().(ClassObjectInternal).lookup(name, cls_attr, attr_orig) - | - cls_attr.isDescriptor() = false and value = cls_attr and origin = attr_orig - or - cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this, value, origin) - ) - } - pragma [noinline] override predicate attributesUnknown() { any() } override predicate subscriptUnknown() { any() } @@ -234,6 +240,12 @@ class SelfInstanceInternal extends TSelfInstance, ObjectInternal { result = lengthFromClass(this.getClass()) } + override predicate initializer(PythonFunctionObjectInternal init, Context callee) { + callee.isRuntime() and + init.getScope() != this.getParameter().getScope() and + this.getClass().attribute("__init__", init, _) + } + } /** Represents a value that has a known class, but no other information */ @@ -245,7 +257,6 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { /** The boolean value of this object, if it has one */ override boolean booleanValue() { - //result = this.getClass().instancesBooleanValue() result = maybe() } From bf78c62594df831a5ff77e636b14e8f3c53dba2f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 26 Apr 2019 15:53:39 +0100 Subject: [PATCH 104/128] Python points-to: Add objects representing missing modules and their attributes. --- .../ql/src/semmle/python/objects/Modules.qll | 153 ++++++++++++++++++ .../src/semmle/python/objects/ObjectAPI.qll | 9 ++ .../semmle/python/objects/ObjectInternal.qll | 6 + .../ql/src/semmle/python/objects/TObject.qll | 45 +++++- .../src/semmle/python/pointsto/PointsTo.qll | 2 +- .../src/semmle/python/security/Exceptions.qll | 8 +- .../PointsTo/absent/Absent.expected | 7 + .../library-tests/PointsTo/absent/Absent.ql | 9 ++ .../library-tests/PointsTo/absent/absent.py | 14 ++ .../library-tests/PointsTo/absent/module.py | 0 .../PointsTo/decorators/Test.expected | 1 + .../PointsTo/general/LocalPointsTo.expected | 1 - .../general/LocalPointsToType.expected | 1 - .../PointsTo/guarded/PointsTo.expected | 2 - .../guarded/PointsToWithType.expected | 2 - .../PointsTo/new/PointsToWithContext.expected | 1 + .../PointsTo/new/Values.expected | 1 + .../pointsto/CallGraphEfficiency.expected | 4 +- .../CallGraphMarginalEfficiency.expected | 4 +- 19 files changed, 252 insertions(+), 18 deletions(-) create mode 100644 python/ql/test/library-tests/PointsTo/absent/Absent.expected create mode 100644 python/ql/test/library-tests/PointsTo/absent/Absent.ql create mode 100644 python/ql/test/library-tests/PointsTo/absent/absent.py create mode 100644 python/ql/test/library-tests/PointsTo/absent/module.py diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 4f94f907753b..4ada9021f890 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -250,3 +250,156 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { } +class AbsentModuleObjectInternal extends ModuleObjectInternal, TAbsentModule { + + override Builtin getBuiltin() { + none() + } + + override string toString() { + if exists(Module m, SyntaxError se | se.getFile() = m.getFile() and m.getName() = this.getName()) then + result = "Unparsable module " + this.getName() + else + result = "Missing module " + this.getName() + } + + override string getName() { + this = TAbsentModule(result) + } + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + missing_imported_module(node, context, this.getName()) + } + + override ClassDecl getClassDeclaration() { + none() + } + + override Module getSourceModule() { + none() + } + + PythonModuleObjectInternal getInitModule() { + none() + } + + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + value = TAbsentModuleAttribute(this, name) and origin = CfgOrigin::unknown() + } + + pragma [noinline] override predicate attributesUnknown() { none() } + + override ControlFlowNode getOrigin() { + none() + } + + override predicate isMissing() { + any() + } + +} + +class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleAttribute { + + override Builtin getBuiltin() { + none() + } + + override string toString() { + exists(ModuleObjectInternal mod, string name | + this = TAbsentModuleAttribute(mod, name) and + result = "Missing module attribute " + mod.getName() + "." + name + ) + } + + override predicate introduced(ControlFlowNode node, PointsToContext context) { + exists(ModuleObjectInternal mod, string name | + this = TAbsentModuleAttribute(mod, name) | + PointsToInternal::pointsTo(node.(AttrNode).getObject(name), context, mod, _) + or + PointsToInternal::pointsTo(node.(ImportMemberNode).getModule(name), context, mod, _) + ) + } + + override ClassDecl getClassDeclaration() { + none() + } + + PythonModuleObjectInternal getInitModule() { + none() + } + + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + pragma [noinline] override predicate attributesUnknown() { any() } + + override ControlFlowNode getOrigin() { + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + // Don't know, assume not callable. + none() + } + + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { + // Don't know, assume not callable. + none() + } + + override boolean isClass() { result = maybe() } + + override boolean isComparable() { result = false } + + override boolean booleanValue() { + result = maybe() + } + + override ObjectInternal getClass() { + result = ObjectInternal::unknownClass() + } + + override boolean isDescriptor() { result = false } + + override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + + override int length() { none() } + + override predicate subscriptUnknown() { any() } + + override predicate isMissing() { + any() + } + +} + diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 1d57656c4ca6..5d3b21eba027 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -56,6 +56,15 @@ class Value extends TObject { predicate isBuiltin() { this.(ObjectInternal).isBuiltin() } + + /** Holds if this value represents an entity that is inferred to exist, + * but missing from the database. + * Most commonly, this is a module that is imported, but wasn't present during extraction. + */ + predicate isMissing() { + this.(ObjectInternal).isMissing() + } + } class ModuleValue extends Value { diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 0765d769f8e4..5d635be32775 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -111,6 +111,12 @@ class ObjectInternal extends TObject { predicate functionAndOffset(CallableObjectInternal function, int offset) { none() } + /** Holds if this 'object' represents an entity that is inferred to exist + * but is missing from the database */ + predicate isMissing() { + none() + } + } diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index a99401aba55a..d01c2e3d763d 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -39,7 +39,9 @@ newtype TObject = exists(moduleNameFromFile(f)) } or - TPythonModule(Module m) { not m.isPackage() } + TPythonModule(Module m) { + not m.isPackage() and not exists(SyntaxError se | se.getFile() = m.getFile()) + } or TTrue() or @@ -160,6 +162,23 @@ newtype TObject = } or TSysVersionInfo() + or + TAbsentModule(string name) { + missing_imported_module(_, _, name) + } + or + TAbsentModuleAttribute(AbsentModuleObjectInternal mod, string attrname) { + ( + PointsToInternal::pointsTo(any(AttrNode attr).getObject(attrname), _, mod, _) + or + PointsToInternal::pointsTo(any(ImportMemberNode imp).getModule(attrname), _, mod, _) + ) + and + exists(string modname | + modname = mod.getName() and + not common_module_name(modname + "." + attrname) + ) + } private predicate is_power_2(int n) { n = 1 or @@ -327,6 +346,30 @@ private predicate neither_class_nor_static_method(Function f) { ) } +predicate missing_imported_module(ControlFlowNode imp, Context ctx, string name) { + ctx.isImport() and imp.(ImportExprNode).getNode().getAnImportedModuleName() = name and + ( + not exists(Module m | m.getName() = name) and + not exists(Builtin b | b.isModule() and b.getName() = name) + or + exists(Module m, SyntaxError se | + m.getName() = name and + se.getFile() = m.getFile() + ) + ) + or + exists(AbsentModuleObjectInternal mod | + PointsToInternal::pointsTo(imp.(ImportMemberNode).getModule(name), ctx, mod, _) and + common_module_name(mod.getName() + "." + name) + ) +} + +predicate common_module_name(string name) { + name = "zope.interface" + or + name = "six.moves" +} + library class ClassDecl extends @py_object { ClassDecl() { diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index d153bdd01381..b7b33be3c85f 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -130,7 +130,7 @@ module PointsTo { PointsToInternal::pointsTo(f, context, value, origin) and cls = value.getClass().getSource() | obj = value.getSource() or - not exists(value.getSource()) and obj = origin + not exists(value.getSource()) and not value.isMissing() and obj = origin ) or /* Backwards compatibility for *args and **kwargs */ diff --git a/python/ql/src/semmle/python/security/Exceptions.qll b/python/ql/src/semmle/python/security/Exceptions.qll index 86a74a1da2f0..d375ab7f33d2 100644 --- a/python/ql/src/semmle/python/security/Exceptions.qll +++ b/python/ql/src/semmle/python/security/Exceptions.qll @@ -7,12 +7,8 @@ import python import semmle.python.security.TaintTracking import semmle.python.security.strings.Basic -private ModuleObject theTracebackModule() { - result.getName() = "traceback" -} - -private FunctionObject traceback_function(string name) { - result = theTracebackModule().attr(name) +private Value traceback_function(string name) { + result = Module::named("traceback").attr(name) } /** diff --git a/python/ql/test/library-tests/PointsTo/absent/Absent.expected b/python/ql/test/library-tests/PointsTo/absent/Absent.expected new file mode 100644 index 000000000000..dc107502efe8 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/absent/Absent.expected @@ -0,0 +1,7 @@ +| absent.py:3:8:3:11 | ControlFlowNode for ImportExpr | Missing module xxxx | +| absent.py:4:1:4:4 | ControlFlowNode for xxxx | Missing module xxxx | +| absent.py:6:6:6:9 | ControlFlowNode for ImportExpr | Missing module xxxx | +| absent.py:6:18:6:21 | ControlFlowNode for ImportMember | Missing module attribute xxxx.open | +| absent.py:8:1:8:4 | ControlFlowNode for open | Missing module attribute xxxx.open | +| absent.py:12:8:12:13 | ControlFlowNode for ImportExpr | Module module | +| absent.py:14:1:14:6 | ControlFlowNode for module | Module module | diff --git a/python/ql/test/library-tests/PointsTo/absent/Absent.ql b/python/ql/test/library-tests/PointsTo/absent/Absent.ql new file mode 100644 index 000000000000..8b690c1d7990 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/absent/Absent.ql @@ -0,0 +1,9 @@ + +import python +import semmle.python.objects.Modules + +from Value val, ControlFlowNode f +where //val = Value::named(name) and +f.pointsTo(val) +select f, val + diff --git a/python/ql/test/library-tests/PointsTo/absent/absent.py b/python/ql/test/library-tests/PointsTo/absent/absent.py new file mode 100644 index 000000000000..5251f2b8b501 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/absent/absent.py @@ -0,0 +1,14 @@ +#There is no xxxx, rely on AbsentModule + +import xxxx +xxxx + +from xxxx import open + +open() + + +#This is be present, so shouldn't be missing +import module + +module diff --git a/python/ql/test/library-tests/PointsTo/absent/module.py b/python/ql/test/library-tests/PointsTo/absent/module.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/ql/test/library-tests/PointsTo/decorators/Test.expected b/python/ql/test/library-tests/PointsTo/decorators/Test.expected index 06068e5206ed..136adce143ff 100644 --- a/python/ql/test/library-tests/PointsTo/decorators/Test.expected +++ b/python/ql/test/library-tests/PointsTo/decorators/Test.expected @@ -1,2 +1,3 @@ | 41 | ControlFlowNode for func1 | Function func1 | test.py:23 | | 42 | ControlFlowNode for func2 | Function wrapper | test.py:10 | +| 43 | ControlFlowNode for func3 | Function wrapper | test.py:17 | diff --git a/python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected b/python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected index 3dbab0c3f102..1960aa00095a 100644 --- a/python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected +++ b/python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected @@ -341,7 +341,6 @@ | 263 | ControlFlowNode for Attribute | int 3 | | 263 | ControlFlowNode for IntegerLiteral | int 3 | | 263 | ControlFlowNode for self | self | -| 264 | ControlFlowNode for Attribute | int 0 | | 264 | ControlFlowNode for Attribute | int 3 | | 264 | ControlFlowNode for self | self | | 267 | ControlFlowNode for Derived4 | class Derived4 | diff --git a/python/ql/test/library-tests/PointsTo/general/LocalPointsToType.expected b/python/ql/test/library-tests/PointsTo/general/LocalPointsToType.expected index 09168c46448d..504263e7c3a9 100644 --- a/python/ql/test/library-tests/PointsTo/general/LocalPointsToType.expected +++ b/python/ql/test/library-tests/PointsTo/general/LocalPointsToType.expected @@ -344,7 +344,6 @@ | 263 | ControlFlowNode for Attribute | int 3 | builtin-class int | | 263 | ControlFlowNode for IntegerLiteral | int 3 | builtin-class int | | 263 | ControlFlowNode for self | self | class G | -| 264 | ControlFlowNode for Attribute | int 0 | builtin-class int | | 264 | ControlFlowNode for Attribute | int 3 | builtin-class int | | 264 | ControlFlowNode for self | self | class G | | 267 | ControlFlowNode for Derived4 | class Derived4 | builtin-class type | diff --git a/python/ql/test/library-tests/PointsTo/guarded/PointsTo.expected b/python/ql/test/library-tests/PointsTo/guarded/PointsTo.expected index 61bb0bc19eb9..b6872c3a4e62 100644 --- a/python/ql/test/library-tests/PointsTo/guarded/PointsTo.expected +++ b/python/ql/test/library-tests/PointsTo/guarded/PointsTo.expected @@ -51,8 +51,6 @@ | test.py | 314 | ControlFlowNode for b | NoneType None | 311 | | test.py | 332 | ControlFlowNode for Attribute | NoneType None | 322 | | test.py | 332 | ControlFlowNode for Attribute | int 4 | 322 | -| test.py | 337 | ControlFlowNode for Attribute | NoneType None | 324 | -| test.py | 337 | ControlFlowNode for Attribute | int 3 | 324 | | test.py | 347 | ControlFlowNode for Attribute | NoneType None | 322 | | test.py | 347 | ControlFlowNode for Attribute | int 4 | 322 | | test.py | 357 | ControlFlowNode for g1 | float 7.0 | 356 | diff --git a/python/ql/test/library-tests/PointsTo/guarded/PointsToWithType.expected b/python/ql/test/library-tests/PointsTo/guarded/PointsToWithType.expected index 21d421b69bd5..2f71fc6107b6 100644 --- a/python/ql/test/library-tests/PointsTo/guarded/PointsToWithType.expected +++ b/python/ql/test/library-tests/PointsTo/guarded/PointsToWithType.expected @@ -51,8 +51,6 @@ | test.py | 314 | ControlFlowNode for b | NoneType None | builtin-class NoneType | 311 | | test.py | 332 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 322 | | test.py | 332 | ControlFlowNode for Attribute | int 4 | builtin-class int | 322 | -| test.py | 337 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 324 | -| test.py | 337 | ControlFlowNode for Attribute | int 3 | builtin-class int | 324 | | test.py | 347 | ControlFlowNode for Attribute | NoneType None | builtin-class NoneType | 322 | | test.py | 347 | ControlFlowNode for Attribute | int 4 | builtin-class int | 322 | | test.py | 357 | ControlFlowNode for g1 | float 7.0 | builtin-class float | 356 | diff --git a/python/ql/test/library-tests/PointsTo/new/PointsToWithContext.expected b/python/ql/test/library-tests/PointsTo/new/PointsToWithContext.expected index b51dcb8d6b35..0a933761e78b 100755 --- a/python/ql/test/library-tests/PointsTo/new/PointsToWithContext.expected +++ b/python/ql/test/library-tests/PointsTo/new/PointsToWithContext.expected @@ -1074,6 +1074,7 @@ WARNING: Predicate points_to has been deprecated and may be removed in future (P | t_type.py:7 | ControlFlowNode for type | builtin-class type | builtin-class type | 7 | import | | t_type.py:7 | ControlFlowNode for type() | builtin-class module | builtin-class type | 7 | import | | t_type.py:9 | ControlFlowNode for type | builtin-class type | builtin-class type | 9 | import | +| t_type.py:9 | ControlFlowNode for type() | *UNKNOWN TYPE* | *UNKNOWN TYPE* | 9 | import | | t_type.py:10 | ControlFlowNode for Dict | Dict | builtin-class dict | 10 | import | | t_type.py:10 | ControlFlowNode for Tuple | Tuple | builtin-class tuple | 10 | import | | t_type.py:10 | ControlFlowNode for object | builtin-class object | builtin-class type | 10 | import | diff --git a/python/ql/test/library-tests/PointsTo/new/Values.expected b/python/ql/test/library-tests/PointsTo/new/Values.expected index c74d14d8db7d..9e2c8da215b1 100644 --- a/python/ql/test/library-tests/PointsTo/new/Values.expected +++ b/python/ql/test/library-tests/PointsTo/new/Values.expected @@ -827,6 +827,7 @@ | t_type.py:7 | ControlFlowNode for sys | import | Module sys | builtin-class module | | t_type.py:7 | ControlFlowNode for type | import | builtin-class type | builtin-class type | | t_type.py:7 | ControlFlowNode for type() | import | builtin-class module | builtin-class type | +| t_type.py:8 | ControlFlowNode for ImportExpr | import | Missing module module | builtin-class module | | t_type.py:9 | ControlFlowNode for type | import | builtin-class type | builtin-class type | | t_type.py:10 | ControlFlowNode for Dict | import | Dict | builtin-class dict | | t_type.py:10 | ControlFlowNode for Tuple | import | (builtin-class object) | builtin-class tuple | diff --git a/python/ql/test/query-tests/analysis/pointsto/CallGraphEfficiency.expected b/python/ql/test/query-tests/analysis/pointsto/CallGraphEfficiency.expected index d72bc6028d6e..cf6df525de2f 100644 --- a/python/ql/test/query-tests/analysis/pointsto/CallGraphEfficiency.expected +++ b/python/ql/test/query-tests/analysis/pointsto/CallGraphEfficiency.expected @@ -1,2 +1,2 @@ -| 0 | 53 | 53 | 100.0 | -| 1 | 3 | 39 | 7.6923076923076925 | +| 0 | 61 | 61 | 100.0 | +| 1 | 5 | 43 | 11.627906976744185 | diff --git a/python/ql/test/query-tests/analysis/pointsto/CallGraphMarginalEfficiency.expected b/python/ql/test/query-tests/analysis/pointsto/CallGraphMarginalEfficiency.expected index d72bc6028d6e..bbdbe208542c 100644 --- a/python/ql/test/query-tests/analysis/pointsto/CallGraphMarginalEfficiency.expected +++ b/python/ql/test/query-tests/analysis/pointsto/CallGraphMarginalEfficiency.expected @@ -1,2 +1,2 @@ -| 0 | 53 | 53 | 100.0 | -| 1 | 3 | 39 | 7.6923076923076925 | +| 0 | 61 | 61 | 100.0 | +| 1 | 3 | 43 | 6.976744186046512 | From 2b72a50dbfd443ee7b0b9b34c24591dbd5001c5e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 15 May 2019 11:24:40 +0100 Subject: [PATCH 105/128] Python points-to: Fix up re extension. --- python/ql/src/semmle/python/types/Extensions.qll | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/ql/src/semmle/python/types/Extensions.qll b/python/ql/src/semmle/python/types/Extensions.qll index 594a1d201b54..8056a52758ba 100644 --- a/python/ql/src/semmle/python/types/Extensions.qll +++ b/python/ql/src/semmle/python/types/Extensions.qll @@ -109,10 +109,10 @@ class ReModulePointToExtension extends PointsToExtension { } override predicate pointsTo(Context context, ObjectInternal value, ControlFlowNode origin) { - value = Module::named("sre_constants").attr("SRE_FLAG_" + name) and - exists(ModuleObjectInternal sre_constants | + exists(ModuleObjectInternal sre_constants, CfgOrigin orig | sre_constants.getName() = "sre_constants" and - sre_constants.attribute(name, value, CfgOrigin::fromCfgNode(origin)) + sre_constants.attribute("SRE_FLAG_" + name, value, orig) and + origin = orig.asCfgNodeOrHere(this) ) and context.appliesTo(this) } From d6d72dcef41e544fba24953cfb979ddce5535ae2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 16 May 2019 11:46:47 +0100 Subject: [PATCH 106/128] Python: Account for --respect-init when determining module name. --- python/ql/src/semmle/python/Module.qll | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/ql/src/semmle/python/Module.qll b/python/ql/src/semmle/python/Module.qll index 9b6b8129e7ad..275071b2925f 100644 --- a/python/ql/src/semmle/python/Module.qll +++ b/python/ql/src/semmle/python/Module.qll @@ -202,11 +202,15 @@ private predicate legalShortName(string name) { } /** Holds if `f` is potentially a source package. - * Does it have an __init__.py file and is it within the source archive? + * Does it have an __init__.py file (or --respect-init=False for Python 2) and is it within the source archive? */ private predicate isPotentialSourcePackage(Folder f) { f.getRelativePath() != "" and - exists(f.getFile("__init__.py")) + ( + exists(f.getFile("__init__.py")) + or + py_flags_versioned("options.respect_init", "False", _) and major_version() = 2 + ) } private string moduleNameFromBase(Container file) { From 2f940d013b447553440ee9ff6fbcd4ded4cc7d4b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 16 May 2019 16:18:54 +0100 Subject: [PATCH 107/128] Python points-to: Fully document object classes. --- .../src/semmle/python/objects/Callables.qll | 60 ++++++++------- .../ql/src/semmle/python/objects/Classes.qll | 73 +++++++++++------- .../src/semmle/python/objects/Constants.qll | 33 ++++---- .../src/semmle/python/objects/Descriptors.qll | 43 ++++++----- .../src/semmle/python/objects/Instances.qll | 13 +++- .../ql/src/semmle/python/objects/Modules.qll | 24 +++--- .../src/semmle/python/objects/ObjectAPI.qll | 77 ++++++++++++++++++- .../semmle/python/objects/ObjectInternal.qll | 44 ++++++++++- .../src/semmle/python/objects/ObjectModel.md | 35 +++++++++ .../src/semmle/python/objects/Sequences.qll | 3 +- .../ql/src/semmle/python/objects/TObject.qll | 16 +++- 11 files changed, 306 insertions(+), 115 deletions(-) create mode 100644 python/ql/src/semmle/python/objects/ObjectModel.md diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 4dec5184181a..53671cee4804 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -11,54 +11,49 @@ private import semmle.python.types.Builtins abstract class CallableObjectInternal extends ObjectInternal { - override int intValue() { - none() - } - - override string strValue() { - none() - } + /** Gets the name of this callable */ + abstract string getName(); - override boolean isClass() { result = false } + /** Gets the scope of this callable if it has one */ + abstract Function getScope(); - /** The boolean value of this object, if it has one */ - override boolean booleanValue() { - result = true - } + /** Gets a call to this callable from the given context */ + abstract CallNode getACall(PointsToContext ctx); - override ClassDecl getClassDeclaration() { - none() - } + /** Gets a call to this callable */ + CallNode getACall() { result = this.getACall(_) } - abstract string getName(); + override boolean isClass() { result = false } - pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - none() - } + override boolean booleanValue() { result = true } - pragma [noinline] override predicate attributesUnknown() { none() } - - abstract Function getScope(); + override ClassDecl getClassDeclaration() { none() } pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } - abstract CallNode getACall(PointsToContext ctx); - - CallNode getACall() { result = this.getACall(_) } - abstract NameNode getParameter(int n); abstract NameNode getParameterByName(string name); + abstract predicate neverReturns(); + override int length() { none() } - abstract predicate neverReturns(); + pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + pragma [noinline] override predicate attributesUnknown() { none() } override predicate subscriptUnknown() { none() } -} + override int intValue() { none() } + + override string strValue() { none() } +} +/** Class representing Python functions */ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFunctionObject { override Function getScope() { @@ -176,6 +171,7 @@ private BasicBlock blockReturningNone(Function func) { } +/** Class representing built-in functions such as `len` or `print`. */ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunctionObject { override Builtin getBuiltin() { @@ -283,7 +279,8 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc } - +/** Class representing methods of built-in classes (otherwise known as method-descriptors) such as `list.append`. + */ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethodObject { override Builtin getBuiltin() { @@ -372,6 +369,11 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod } +/** Class representing bound-methods. + * Note that built-in methods, such as `[].append` are also represented as bound-methods. + * Although built-in methods and bound-methods are distinct classes in CPython, their behaviour + * is the same and we treat them identically. + */ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { override Builtin getBuiltin() { diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 48a362665ad4..abe363617adc 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -8,23 +8,9 @@ private import semmle.python.pointsto.PointsToContext private import semmle.python.pointsto.MRO private import semmle.python.types.Builtins - +/** Class representing classes */ abstract class ClassObjectInternal extends ObjectInternal { - override boolean booleanValue() { - result = true - } - - override boolean isClass() { result = true } - - override int intValue() { - none() - } - - override string strValue() { - none() - } - string getName() { result = this.getClassDeclaration().getName() } @@ -33,6 +19,33 @@ abstract class ClassObjectInternal extends ObjectInternal { result = Types::getMro(this).isSpecial() } + /** Looks up the attribute `name` on this class. + * Note that this may be different from `this.attr(name)`. + * For example given the class: + * ```class C: + * @classmethod + * def f(cls): pass + * ``` + * `this.lookup("f")` is equivent to `C.__dict__['f']`, which is the class-method + * whereas + * `this.attr("f") is equivalent to `C.f`, which is a bound-method. + */ + abstract predicate lookup(string name, ObjectInternal value, CfgOrigin origin); + + /** Holds if this is a subclass of the `Iterable` abstract base class. */ + boolean isIterableSubclass() { + this = ObjectInternal::builtin("list") and result = true + or + this = ObjectInternal::builtin("set") and result = true + or + this = ObjectInternal::builtin("dict") and result = true + or + this != ObjectInternal::builtin("list") and + this != ObjectInternal::builtin("set") and + this != ObjectInternal::builtin("dict") and + result = false + } + override boolean isDescriptor() { result = false } pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } @@ -46,8 +59,6 @@ abstract class ClassObjectInternal extends ObjectInternal { descriptor.isDescriptor() = true } - abstract predicate lookup(string name, ObjectInternal value, CfgOrigin origin); - /** Approximation to descriptor protocol, skipping meta-descriptor protocol */ pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { exists(ObjectInternal descriptor, CfgOrigin desc_origin | @@ -62,24 +73,25 @@ abstract class ClassObjectInternal extends ObjectInternal { override int length() { none() } - boolean isIterableSubclass() { - this = ObjectInternal::builtin("list") and result = true - or - this = ObjectInternal::builtin("set") and result = true - or - this = ObjectInternal::builtin("dict") and result = true - or - this != ObjectInternal::builtin("list") and - this != ObjectInternal::builtin("set") and - this != ObjectInternal::builtin("dict") and - result = false + override boolean booleanValue() { result = true } + + override boolean isClass() { result = true } + + override int intValue() { + none() + } + + override string strValue() { + none() } override predicate subscriptUnknown() { none() } } +/** Class representing Python source classes */ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject { + /** Gets the scope for this Python class */ Class getScope() { exists(ClassExpr expr | this = TPythonClassObject(expr.getAFlowNode()) and @@ -141,6 +153,7 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject } +/** Class representing built-in classes, except `type` */ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObject { override Builtin getBuiltin() { @@ -194,7 +207,7 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec } - +/** A class representing an unknown class */ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { override string toString() { @@ -243,6 +256,7 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { } +/** A class representing the built-in class `type`. */ class TypeInternal extends ClassObjectInternal, TType { override string toString() { @@ -292,6 +306,7 @@ class TypeInternal extends ClassObjectInternal, TType { } +/** A class representing a dynamically created class `type(name, *args, **kwargs)`. */ class DynamicallyCreatedClass extends ClassObjectInternal, TDynamicClass { override string toString() { diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index 48163dd050e9..4b8d00df0462 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -8,7 +8,10 @@ private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins - +/** Class representing constants. + * Includes `None`, `True` and `False` as + * well as strings and integers. + */ abstract class ConstantObjectInternal extends ObjectInternal { override ClassDecl getClassDeclaration() { @@ -66,7 +69,7 @@ abstract class ConstantObjectInternal extends ObjectInternal { } -abstract class BooleanObjectInternal extends ConstantObjectInternal { +private abstract class BooleanObjectInternal extends ConstantObjectInternal { BooleanObjectInternal() { this = TTrue() or this = TFalse() @@ -79,9 +82,13 @@ abstract class BooleanObjectInternal extends ConstantObjectInternal { override int length() { none() } + override string strValue() { + none() + } + } -class TrueObjectInternal extends BooleanObjectInternal, TTrue { +private class TrueObjectInternal extends BooleanObjectInternal, TTrue { override string toString() { result = "bool True" @@ -99,17 +106,13 @@ class TrueObjectInternal extends BooleanObjectInternal, TTrue { result = 1 } - override string strValue() { - none() - } - override Builtin getBuiltin() { result = Builtin::special("True") } } -class FalseObjectInternal extends BooleanObjectInternal, TFalse { +private class FalseObjectInternal extends BooleanObjectInternal, TFalse { override string toString() { result = "bool False" @@ -127,17 +130,13 @@ class FalseObjectInternal extends BooleanObjectInternal, TFalse { result = 0 } - override string strValue() { - none() - } - override Builtin getBuiltin() { result = Builtin::special("False") } } -class NoneObjectInternal extends ConstantObjectInternal, TNone { +private class NoneObjectInternal extends ConstantObjectInternal, TNone { override string toString() { result = "None" @@ -172,7 +171,7 @@ class NoneObjectInternal extends ConstantObjectInternal, TNone { } -class IntObjectInternal extends ConstantObjectInternal, TInt { +private class IntObjectInternal extends ConstantObjectInternal, TInt { override string toString() { result = "int " + this.intValue().toString() @@ -209,7 +208,7 @@ class IntObjectInternal extends ConstantObjectInternal, TInt { } -class FloatObjectInternal extends ConstantObjectInternal, TFloat { +private class FloatObjectInternal extends ConstantObjectInternal, TFloat { override string toString() { if this.floatValue() = this.floatValue().floor() then ( @@ -255,7 +254,7 @@ class FloatObjectInternal extends ConstantObjectInternal, TFloat { } -class UnicodeObjectInternal extends ConstantObjectInternal, TUnicode { +private class UnicodeObjectInternal extends ConstantObjectInternal, TUnicode { override string toString() { result = "'" + this.strValue() + "'" @@ -296,7 +295,7 @@ class UnicodeObjectInternal extends ConstantObjectInternal, TUnicode { } -class BytesObjectInternal extends ConstantObjectInternal, TBytes { +private class BytesObjectInternal extends ConstantObjectInternal, TBytes { override string toString() { result = "'" + this.strValue() + "'" diff --git a/python/ql/src/semmle/python/objects/Descriptors.qll b/python/ql/src/semmle/python/objects/Descriptors.qll index 914311c2eff4..4b3295028004 100644 --- a/python/ql/src/semmle/python/objects/Descriptors.qll +++ b/python/ql/src/semmle/python/objects/Descriptors.qll @@ -7,8 +7,30 @@ private import semmle.python.pointsto.PointsToContext private import semmle.python.pointsto.MRO private import semmle.python.types.Builtins +/** Class representing property objects in Python */ class PropertyInternal extends ObjectInternal, TProperty { + /** Gets the name of this property */ + string getName() { + result = this.getGetter().getName() + } + + /** Gets the getter function of this property */ + CallableObjectInternal getGetter() { + this = TProperty(_, _, result) + } + + /** Gets the setter function of this property */ + CallableObjectInternal getSetter() { + exists(CallNode call, AttrNode setter | + call.getFunction() = setter and + PointsToInternal::pointsTo(setter.getObject("setter"), this.getContext(), this, _) and + PointsToInternal::pointsTo(call.getArg(0), this.getContext(), result, _) + ) + } + + private Context getContext() { this = TProperty(_,result, _) } + override string toString() { result = "property" + this.getName() } @@ -25,18 +47,10 @@ class PropertyInternal extends ObjectInternal, TProperty { override ObjectInternal getClass() { result = ObjectInternal::property() } - CallableObjectInternal getGetter() { - this = TProperty(_, _, result) - } - override boolean isComparable() { result = true } override Builtin getBuiltin() { none() } - string getName() { - result = this.getGetter().getName() - } - override ControlFlowNode getOrigin() { this = TProperty(result, _, _) } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -58,17 +72,6 @@ class PropertyInternal extends ObjectInternal, TProperty { override boolean isDescriptor() { result = true } override int length() { none() } - - private Context getContext() { this = TProperty(_,result, _) } - - CallableObjectInternal getSetter() { - exists(CallNode call, AttrNode setter | - call.getFunction() = setter and - PointsToInternal::pointsTo(setter.getObject("setter"), this.getContext(), this, _) and - PointsToInternal::pointsTo(call.getArg(0), this.getContext(), result, _) - ) - } - pragma [noinline] override predicate binds(ObjectInternal cls, string name, ObjectInternal descriptor) { none() } pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { @@ -90,6 +93,7 @@ class PropertyInternal extends ObjectInternal, TProperty { } +/** A class representing classmethods in Python */ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { override string toString() { @@ -105,6 +109,7 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { ) } + /** Gets the function wrapped by this classmethod object */ CallableObjectInternal getFunction() { this = TClassMethod(_, result) } diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 7f8d6498f817..bfd95aa274eb 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -8,6 +8,7 @@ private import semmle.python.pointsto.MRO private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins +/** A class representing instances */ abstract class InstanceObject extends ObjectInternal { pragma [nomagic] @@ -36,6 +37,7 @@ abstract class InstanceObject extends ObjectInternal { ) } + /** Holds if `init` in the context `callee` is the initializer of this instance */ abstract predicate initializer(PythonFunctionObjectInternal init, Context callee); } @@ -46,6 +48,9 @@ private predicate self_variable_reaching_init_exit(EssaVariable self) { self.getScope().getName() = "__init__" } +/** A class representing instances instantiated at a specific point in the program (statically) + * For example the code `C()` would be a specific instance of `C`. + */ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject { override string toString() { @@ -147,7 +152,8 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject { } - +/** A class representing context-free instances represented by `self` in the source code + */ class SelfInstanceInternal extends TSelfInstance, InstanceObject { override string toString() { @@ -248,7 +254,7 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject { } -/** Represents a value that has a known class, but no other information */ +/** A class representing a value that has a known class, but no other information */ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { override string toString() { @@ -360,6 +366,7 @@ private predicate cls_descriptor(ClassObjectInternal cls, string name, ObjectInt descriptor.isDescriptor() = true } +/** A class representing an instance of the `super` class */ class SuperInstance extends TSuperInstance, ObjectInternal { override string toString() { @@ -375,10 +382,12 @@ class SuperInstance extends TSuperInstance, ObjectInternal { ) } + /** Gets the class declared as the starting point for MRO lookup. */ ClassObjectInternal getStartClass() { this = TSuperInstance(_, result) } + /** Gets 'self' object */ ObjectInternal getSelf() { this = TSuperInstance(result, _) } diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 4ada9021f890..d47966238972 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -7,10 +7,13 @@ private import semmle.python.pointsto.MRO private import semmle.python.pointsto.PointsToContext private import semmle.python.types.Builtins +/** A class representing modules */ abstract class ModuleObjectInternal extends ObjectInternal { + /** Gets the name of this module */ abstract string getName(); + /** Gets the source scope of this module, if it has one. */ abstract Module getSourceModule(); override predicate callResult(ObjectInternal obj, CfgOrigin origin) { @@ -47,12 +50,14 @@ abstract class ModuleObjectInternal extends ObjectInternal { override predicate subscriptUnknown() { any() } + /** Holds if this module is a `__init__.py` module. */ predicate isInitModule() { any(PackageObjectInternal package).getInitModule() = this } } +/** A class representing built-in modules */ class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleObject { override Builtin getBuiltin() { @@ -104,6 +109,7 @@ class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleOb } +/** A class representing packages */ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { override Builtin getBuiltin() { @@ -114,6 +120,7 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { result = "Package " + this.getName() } + /** Gets the folder for this package */ Folder getFolder() { this = TPackageObject(result) } @@ -134,6 +141,7 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { result.getFile() = this.getFolder().getFile("__init__.py") } + /** Gets the init module of this package */ PythonModuleObjectInternal getInitModule() { result = TPythonModule(this.getSourceModule()) } @@ -145,6 +153,7 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { ) } + /** Gets the submodule `name` of this package */ ModuleObjectInternal submodule(string name) { result.getName() = this.getName() + "." + name } @@ -196,6 +205,7 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { } +/** A class representing Python modules */ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { override Builtin getBuiltin() { @@ -222,10 +232,6 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { this = TPythonModule(result) } - PythonModuleObjectInternal getInitModule() { - result = TPythonModule(this.getSourceModule()) - } - override int intValue() { none() } @@ -250,6 +256,7 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { } +/** A class representing a module that is missing from the DB, but inferred to exists from imports. */ class AbsentModuleObjectInternal extends ModuleObjectInternal, TAbsentModule { override Builtin getBuiltin() { @@ -279,10 +286,6 @@ class AbsentModuleObjectInternal extends ModuleObjectInternal, TAbsentModule { none() } - PythonModuleObjectInternal getInitModule() { - none() - } - override int intValue() { none() } @@ -311,6 +314,7 @@ class AbsentModuleObjectInternal extends ModuleObjectInternal, TAbsentModule { } +/** A class representing an attribute of a missing module. */ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleAttribute { override Builtin getBuiltin() { @@ -337,10 +341,6 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA none() } - PythonModuleObjectInternal getInitModule() { - none() - } - override int intValue() { none() } diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 5d3b21eba027..b2a314cc40af 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -1,11 +1,25 @@ +/** + * Public API for "objects" + * A `Value` is a static approximation to a set of runtime objects. + */ + + + + import python private import TObject private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext +/* Use the term `ObjectSource` to refer to DB entity. Either a CFG node + * for Python objects, or `@py_cobject` entity for built-in objects. + */ class ObjectSource = Object; +/** Class representing values in the Python program + * Each `Value` is a static approximation to a set of one or more real objects. + */ class Value extends TObject { Value() { @@ -18,14 +32,20 @@ class Value extends TObject { result = this.(ObjectInternal).toString() } + /** Gets a `ControlFlowNode` that refers to this object. */ ControlFlowNode getAReference() { PointsToInternal::pointsTo(result, _, this, _) } + /** Gets the class of this object. + * Strictly, the `Value` representing the class of the objects + * represented by this Value. + */ Value getClass() { result = this.(ObjectInternal).getClass() } + /** Gets a call to this object */ CallNode getACall() { PointsToInternal::pointsTo(result.getFunction(), _, this, _) or @@ -35,6 +55,7 @@ class Value extends TObject { ) } + /** Gets a call to this object with the given `caller` context. */ CallNode getACall(PointsToContext caller) { PointsToInternal::pointsTo(result.getFunction(), caller, this, _) or @@ -44,15 +65,21 @@ class Value extends TObject { ) } + /** Gets a `Value` that represents the attribute `name` of this object. */ Value attr(string name) { this.(ObjectInternal).attribute(name, result, _) } - /* For backwards compatibility with old API */ + /** DEPRECATED: For backwards compatibility with old API + * Use `Value` instead of `ObjectSource`. + */ deprecated ObjectSource getSource() { result = this.(ObjectInternal).getSource() } + /** Holds if this value is builtin. Applies to built-in functions and methods, + * but also integers and strings. + */ predicate isBuiltin() { this.(ObjectInternal).isBuiltin() } @@ -67,12 +94,20 @@ class Value extends TObject { } +/** Class representing modules in the Python program + * Each `ModuleValue` represents a module object in the Python program. + */ class ModuleValue extends Value { ModuleValue() { this instanceof ModuleObjectInternal } + /** Holds if this module "exports" name. + * That is, does it define `name` in `__all__` or is + * `__all__` not defined and `name` a global variable that does not start with "_" + * This is the set of names imported by `from ... import *`. + */ predicate exports(string name) { not this.(ModuleObjectInternal).attribute("__all__", _, _) and exists(this.attr(name)) and not name.charAt(0) = "_" @@ -80,10 +115,12 @@ class ModuleValue extends Value { py_exports(this.getScope(), name) } + /** Gets the name of this module */ string getName() { result = this.(ModuleObjectInternal).getName() } + /** Gets the scope for this module, provided that it is a Python module. */ Module getScope() { result = this.(ModuleObjectInternal).getSourceModule() } @@ -92,6 +129,7 @@ class ModuleValue extends Value { module Module { + /** Gets the `ModuleValue` named `name` */ ModuleValue named(string name) { result.getName() = name } @@ -100,6 +138,16 @@ module Module { module Value { + /** Gets the `Value` named `name`. + * If has at least one '.' in `name`, then the part of + * the name to the left of the rightmost '.' is interpreted as a module name + * and the part after the rightmost '.' as an attribute of that module. + * For example, `Value::named("os.path.join")` is the `Value` representing the function + * `join` in the module `os.path`. + * If there is no '.' in `name`, then the `Value` returned is the builtin + * object of that name. + * For example `Value::named("len")` is the `Value` representing the `len` built-in function. + */ Value named(string name) { exists(string modname, string attrname | name = modname + "." + attrname | @@ -111,28 +159,40 @@ module Value { } +/** Class representing callables in the Python program + * Callables include Python functions, built-in functions and bound-methods, + * but not classes. + */ class CallableValue extends Value { CallableValue() { this instanceof CallableObjectInternal } + /** Holds if this callable never returns once called. + * For example, `sys.exit` + */ predicate neverReturns() { this.(CallableObjectInternal).neverReturns() } + + /** Gets the scope for this function, provided that it is a Python function. */ Function getScope() { result = this.(PythonFunctionObjectInternal).getScope() } + /** Gets the `n`th parameter node of this callable. */ NameNode getParameter(int n) { result = this.(CallableObjectInternal).getParameter(n) } + /** Gets the `name`d parameter node of this callable. */ NameNode getParameterByName(string name) { result = this.(CallableObjectInternal).getParameterByName(name) } + /** Gets the argument corresponding to the `n'th parameter node of this callable. */ ControlFlowNode getArgumentForCall(CallNode call, int n) { exists(ObjectInternal called, int offset | PointsToInternal::pointsTo(call.getFunction(), _, called, _) and @@ -147,6 +207,8 @@ class CallableValue extends Value { ) } + + /** Gets the argument corresponding to the `name`d parameter node of this callable. */ ControlFlowNode getNamedArgumentForCall(CallNode call, string name) { exists(CallableObjectInternal called, int offset | PointsToInternal::pointsTo(call.getFunction(), _, called, _) and @@ -167,6 +229,8 @@ class CallableValue extends Value { } +/** Class representing classes in the Python program, both Python and built-in. + */ class ClassValue extends Value { ClassValue() { @@ -178,6 +242,17 @@ class ClassValue extends Value { result = Types::getMro(this).getAnItem() } + /** Looks up the attribute `name` on this class. + * Note that this may be different from `this.attr(name)`. + * For example given the class: + * ```class C: + * @classmethod + * def f(cls): pass + * ``` + * `this.lookup("f")` is equivent to `C.__dict__['f']`, which is the class-method + * whereas + * `this.attr("f") is equivalent to `C.f`, which is a bound-method. + */ Value lookup(string name) { this.(ClassObjectInternal).lookup(name, result, _) } diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 5d635be32775..a2f5a950b5ae 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -1,4 +1,7 @@ -import python +/** + * Internal object API. + * For use by points-to and testing only. + */ private import semmle.python.objects.TObject private import semmle.python.pointsto.PointsTo @@ -13,6 +16,7 @@ import semmle.python.objects.Constants import semmle.python.objects.Sequences import semmle.python.objects.Descriptors + class ObjectInternal extends TObject { abstract string toString(); @@ -21,14 +25,23 @@ class ObjectInternal extends TObject { * true and false if the "object" represents a set of possible objects. */ abstract boolean booleanValue(); + /** Holds if this object is introduced into the code base at `node` given the `context` + * This means that `node`, in `context`, points-to this object, but the object has not flowed + * there from anywhere else. + * Examples: + * * The object `None` is "introduced" by the keyword "None". + * * A bound method would be "introduced" when relevant attribute on an instance + * is accessed. In `x = X(); x.m` `x.m` introduces the bound method. + */ abstract predicate introduced(ControlFlowNode node, PointsToContext context); - /** Gets the class declaration for this object, if it is a declared class. */ + /** Gets the class declaration for this object, if it is a class with a declaration. */ abstract ClassDecl getClassDeclaration(); /** True if this "object" is a class. */ abstract boolean isClass(); + /** Holds if this object is a class. That is, it's class inherits from `type` */ abstract ObjectInternal getClass(); /** True if this "object" can be meaningfully analysed for @@ -44,7 +57,8 @@ class ObjectInternal extends TObject { abstract Builtin getBuiltin(); /** Gets a control flow node that represents the source origin of this - * objects. + * objects. Source code objects should attempt to return + * exactly one result for this method. */ abstract ControlFlowNode getOrigin(); @@ -68,16 +82,27 @@ class ObjectInternal extends TObject { */ abstract string strValue(); + /** Holds if the function `scope` is called when this object is called and `paramOffset` + * is the difference from the parameter position and the argument position. + * For a normal function `paramOffset` is 0. For classes and bound-methods it is 1. + */ abstract predicate calleeAndOffset(Function scope, int paramOffset); final predicate isBuiltin() { exists(this.getBuiltin()) } + /** Holds if the result of getting the attribute `name` is `value` and that `value` comes + * from `origin`. Note this is *not* the same as class lookup. For example + * for an object `x` the attribute `name` (`x.name`) may refer to a bound-method, an attribute of the + * instance, or an attribute of the class. + */ abstract predicate attribute(string name, ObjectInternal value, CfgOrigin origin); + /** Holds if the attributes of this object are wholy or partly unknowable */ abstract predicate attributesUnknown(); + /** Holds if the result of subscripting this object are wholy or partly unknowable */ abstract predicate subscriptUnknown(); /** For backwards compatibility shim -- Not all objects have a "source". @@ -90,11 +115,20 @@ class ObjectInternal extends TObject { result = this.getBuiltin() } + /** Holds if this object is a descriptor. + * Holds, for example, for functions and properties and not for integers. + */ abstract boolean isDescriptor(); + /** Holds if the result of attribute access on the class holding this descriptor is `value`, originating at `origin` + * For example, although `T.__dict__['name'] = classmethod(f)`, `T.name` is a bound-method, binding `f` and `T` + */ pragma[nomagic] abstract predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin); + /** Holds if the result of attribute access on an instance of a class holding this descriptor is `value`, originating at `origin` + * For example, with `T.__dict__['name'] = classmethod(f)`, `T().name` is a bound-method, binding `f` and `T` + */ pragma[nomagic] abstract predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin); @@ -109,6 +143,10 @@ class ObjectInternal extends TObject { */ abstract int length(); + /** Holds if the object `function` is called when this object is called and `paramOffset` + * is the difference from the parameter position and the argument position. + * For a normal function `paramOffset` is 0. For classes and bound-methods it is 1. + */ predicate functionAndOffset(CallableObjectInternal function, int offset) { none() } /** Holds if this 'object' represents an entity that is inferred to exist diff --git a/python/ql/src/semmle/python/objects/ObjectModel.md b/python/ql/src/semmle/python/objects/ObjectModel.md new file mode 100644 index 000000000000..bf818c6b97ec --- /dev/null +++ b/python/ql/src/semmle/python/objects/ObjectModel.md @@ -0,0 +1,35 @@ + +# Object model for Python analysis + +## General idea + +Each 'object' in the analysis represents a static approximation to a set of objects in the actual program. +For objects like classes and functions there is a (mostly) one-to-one correspondence. +For instances, bound-methods and other short lived objects, one entity in the analysis represents a set of similar objects. + +## APIs + +Objects have two APIs; an internal and a user-facing API. + +### Internal API + +The internal API, exposed through `ObjectInternal` class, provides the predicates necessary for points-to to infer the behaviour of the object. This covers a number of operations: + + * Truth tests + * Type tests and type(x) calls + * Attribute access + * Calls + * Subscripting + * Descriptors + * Comparing objects + * Treating the object as an integer or a string + +Part of internal API exists to allow other objects to implement the points-to facing part of the API. +For example, behaviour of a (Python) class will determine the behaviour of instances of that class. + +### User-facing API + +The user-facing API, exposed through `Value` class, provides a higher level API designed for writing queries rather +than implementing points-to. It provides easy access to objects by name and the ability to find calls to, attributes of, and references to objects. + + diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index f7d08a523ab5..cb686d4b94b6 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -13,6 +13,7 @@ private import semmle.python.types.Builtins abstract class SequenceObjectInternal extends ObjectInternal { + /** Gets the `n`th item of this sequence, if one exists. */ abstract ObjectInternal getItem(int n); /** The boolean value of this object, this may be both @@ -41,7 +42,7 @@ abstract class TupleObjectInternal extends SequenceObjectInternal { result = "(" + this.contents(0) + ")" } - string contents(int n) { + private string contents(int n) { n = this.length() - 1 and result = this.getItem(n).toString() or result = this.getItem(n).toString() + ", " + this.contents(n+1) diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index d01c2e3d763d..8de2f939a20e 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -304,8 +304,7 @@ private predicate self_parameter(ParameterDefinition def, PointsToContext contex ) } -/** INTERNAL -- Use `not cls.isAbstract()` instead. */ -cached predicate concrete_class(PythonClassObjectInternal cls) { +private cached predicate concrete_class(PythonClassObjectInternal cls) { cls.getClass() != abcMetaClassObject() or exists(Class c | @@ -364,12 +363,19 @@ predicate missing_imported_module(ControlFlowNode imp, Context ctx, string name) ) } +/* Helper for missing modules to determine if name `x.y` is a module `x.y` or + * an attribute `y` of module `x`. This list should be added to as required. + */ predicate common_module_name(string name) { name = "zope.interface" or name = "six.moves" } +/** A declaration of a class, either a built-in class or a source definition + * This acts as a helper for ClassObjectInternal allowing some lookup without + * recursion. + */ library class ClassDecl extends @py_object { ClassDecl() { @@ -382,16 +388,19 @@ library class ClassDecl extends @py_object { result = "ClassDecl" } + /** Gets the class scope for Python class declarations */ Class getClass() { result = this.(ControlFlowNode).getNode().(ClassExpr).getInnerScope() } + /** Holds if this class declares the attribute `name` */ predicate declaresAttribute(string name) { exists(this.(Builtin).getMember(name)) or exists(SsaVariable var | name = var.getId() and var.getAUse() = this.getClass().getANormalExit()) } + /** Gets the name of this class */ string getName() { result = this.(Builtin).getName() or @@ -423,6 +432,7 @@ library class ClassDecl extends @py_object { this = Builtin::builtin("float") } + /** Holds if for class `C`, `C()` returns an instance of `C` */ predicate callReturnsInstance() { exists(Class pycls | pycls = this.getClass() | @@ -440,6 +450,7 @@ library class ClassDecl extends @py_object { this instanceof Builtin } + /** Holds if this class is the abstract base class */ predicate isAbstractBaseClass(string name) { exists(Module m | m.getName() = "_abcoll" @@ -450,5 +461,6 @@ library class ClassDecl extends @py_object { this.getName() = name ) } + } From 8558cb3f2f40b5239ef3c6ed2931164ee49baaeb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 16 May 2019 16:25:48 +0100 Subject: [PATCH 108/128] Python points-to: Update Overview.qll to reflect new object model. --- python/ql/src/semmle/python/pointsto/Overview.qll | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/ql/src/semmle/python/pointsto/Overview.qll b/python/ql/src/semmle/python/pointsto/Overview.qll index f46f83dbb3c0..c92285ab7b88 100644 --- a/python/ql/src/semmle/python/pointsto/Overview.qll +++ b/python/ql/src/semmle/python/pointsto/Overview.qll @@ -44,8 +44,11 @@ * * Functions (both in the source and builtin) * * Literal constants defined in the source (string and numbers) * * Constant objects defined in compiled libraries and the interpreter (None, boolean, strings and numbers) - * * Some calls (many calls are absent as we can infer what the call returns). Consider a call to represent the set of objects that it could return. - * * Some other constructs that might create a new object. + * * A few other constants such as small integers. + * * Instances of classes + * * Bound methods, static- and class-methods, and properties. + * * Instances of `super`. + * * Missing modules, where no concrete module is found for an import. * * A number of constructs that might create a new object, such as binary operations, are omitted if there is no useful information to can be attached to them and they would just increase the size of the database. * From 685826e436220fa9ce1d4d07bcb5e2b21c57d538 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 17 May 2019 14:04:57 +0100 Subject: [PATCH 109/128] Python points-to: Prevent bad magic. --- python/ql/src/semmle/python/objects/ObjectInternal.qll | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index a2f5a950b5ae..deaa017db63e 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -97,6 +97,7 @@ class ObjectInternal extends TObject { * for an object `x` the attribute `name` (`x.name`) may refer to a bound-method, an attribute of the * instance, or an attribute of the class. */ + pragma [nomagic] abstract predicate attribute(string name, ObjectInternal value, CfgOrigin origin); /** Holds if the attributes of this object are wholy or partly unknowable */ From f057d5cb6b4a53476127a0e1cc16dadb0a87153a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 May 2019 15:31:41 +0100 Subject: [PATCH 110/128] Python: Speed up MRO calculation a bit. --- python/ql/src/semmle/python/pointsto/MRO.qll | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python/ql/src/semmle/python/pointsto/MRO.qll b/python/ql/src/semmle/python/pointsto/MRO.qll index 2b3e7fcd3b09..7aabc29ebd4e 100644 --- a/python/ql/src/semmle/python/pointsto/MRO.qll +++ b/python/ql/src/semmle/python/pointsto/MRO.qll @@ -33,6 +33,8 @@ cached private newtype TClassList = Empty() /* Keep ClassList finite and as small as possible */ private predicate required_cons(ClassObjectInternal head, ClassList tail) { + tail = Mro::newStyleMro(sole_base(head)) + or tail = merge_of_linearization_of_bases(head) or exists(ClassObjectInternal cls, int n | @@ -61,6 +63,11 @@ private predicate required_cons(ClassObjectInternal head, ClassList tail) { tail = list_old_style_base_mros(head).flatten() } +private ClassObjectInternal sole_base(ClassObjectInternal cls) { + Types::base_count(cls) = 1 and + result = Types::getBase(cls, 0) +} + /** A list of classes, used to represent the MRO of a class */ class ClassList extends TClassList { @@ -413,7 +420,7 @@ private ClassListList list_of_linearization_of_bases_plus_bases(ClassObjectInter } private ClassListList list_of_linearization_of_bases_plus_bases(ClassObjectInternal cls, int n) { - result = ConsList(bases(cls), EmptyList()) and n = Types::base_count(cls) + result = ConsList(bases(cls), EmptyList()) and n = Types::base_count(cls) and n > 1 or exists(ClassListList partial | partial = list_of_linearization_of_bases_plus_bases(cls, n+1) and @@ -476,6 +483,8 @@ module Mro { cls = ObjectInternal::builtin("object") and result = Cons(cls, Empty()) or result = Cons(cls, merge_of_linearization_of_bases(cls)) + or + result = Cons(cls, newStyleMro(sole_base(cls))) } cached ClassList oldStyleMro(ClassObjectInternal cls) { From 1018f27815529534bcb2e5656129767348f48ad0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 May 2019 12:58:06 +0100 Subject: [PATCH 111/128] Python points-to: Add more comments about TObject and ObjectInternal. --- .../semmle/python/objects/ObjectInternal.qll | 17 ++++----- .../ql/src/semmle/python/objects/TObject.qll | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index deaa017db63e..dc1a4ee5c0aa 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -38,15 +38,14 @@ class ObjectInternal extends TObject { /** Gets the class declaration for this object, if it is a class with a declaration. */ abstract ClassDecl getClassDeclaration(); - /** True if this "object" is a class. */ + /** True if this "object" is a class. That is, its class inherits from `type` */ abstract boolean isClass(); - /** Holds if this object is a class. That is, it's class inherits from `type` */ + /** Gets the class of this object. */ abstract ObjectInternal getClass(); - /** True if this "object" can be meaningfully analysed for - * truth or false in comparisons. For example, `None` or `int` can be, but `int()` - * or an unknown string cannot. + /** True if this "object" can be meaningfully analysed to determine its boolean value in comparisons. + * For example, `None` or `int` can be, but `int()` or an unknown string cannot. */ abstract boolean isComparable(); @@ -57,7 +56,7 @@ class ObjectInternal extends TObject { abstract Builtin getBuiltin(); /** Gets a control flow node that represents the source origin of this - * objects. Source code objects should attempt to return + * object. Source code objects should attempt to return * exactly one result for this method. */ abstract ControlFlowNode getOrigin(); @@ -100,10 +99,10 @@ class ObjectInternal extends TObject { pragma [nomagic] abstract predicate attribute(string name, ObjectInternal value, CfgOrigin origin); - /** Holds if the attributes of this object are wholy or partly unknowable */ + /** Holds if the attributes of this object are wholly or partly unknowable */ abstract predicate attributesUnknown(); - /** Holds if the result of subscripting this object are wholy or partly unknowable */ + /** Holds if the result of subscripting this object are wholly or partly unknowable */ abstract predicate subscriptUnknown(); /** For backwards compatibility shim -- Not all objects have a "source". @@ -140,7 +139,7 @@ class ObjectInternal extends TObject { abstract predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor); /** Gets the length of the sequence that this "object" represents. - * Always returns a value for a sequence, will be -1 if object has no fixed length. + * Always returns a value for a sequence, will be -1 if the object has no fixed length. */ abstract int length(); diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 8de2f939a20e..3a60947a9f1d 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -4,19 +4,27 @@ private import semmle.python.objects.ObjectInternal private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext +/** Internal type backing `ObjectInternal` and `Value` + * See `ObjectInternal.qll` for an explanation of the API. + */ newtype TObject = + /* Builtin class objects */ TBuiltinClassObject(Builtin bltn) { bltn.isClass() and not bltn = Builtin::unknownType() and not bltn = Builtin::special("type") } or + /* Builtin function objects (module members) */ TBuiltinFunctionObject(Builtin bltn) { bltn.isFunction() } or + /* Builtin method objects (class members) */ TBuiltinMethodObject(Builtin bltn) { bltn.isMethod() } or + /* Builtin module objects */ TBuiltinModuleObject(Builtin bltn) { bltn.isModule() } or + /* Other builtin objects from the interpreter */ TBuiltinOpaqueObject(Builtin bltn) { not bltn.isClass() and not bltn.isFunction() and not bltn.isMethod() and not bltn.isModule() and @@ -27,34 +35,45 @@ newtype TObject = not py_special_objects(bltn, _) } or + /* Python function objects (including lambdas) */ TPythonFunctionObject(ControlFlowNode callable) { callable.getNode() instanceof CallableExpr } or + /* Python class objects */ TPythonClassObject(ControlFlowNode classexpr) { classexpr.getNode() instanceof ClassExpr } or + /* Package objects */ TPackageObject(Folder f) { exists(moduleNameFromFile(f)) } or + /* Python module objects */ TPythonModule(Module m) { not m.isPackage() and not exists(SyntaxError se | se.getFile() = m.getFile()) } or + /* `True` */ TTrue() or + /* `False` */ TFalse() or + /* `None` */ TNone() or + /* Represents any value about which nothing useful is known */ TUnknown() or + /* Represents any value known to be a class, but not known to be any specific class */ TUnknownClass() or + /* Represents the absence of a value. Used by points-to for tracking undefined variables */ TUndefined() or + /* The integer `n` */ TInt(int n) { // Powers of 2 are used for flags is_power_2(n) or @@ -70,10 +89,12 @@ newtype TObject = n = any(Builtin b).intValue() } or + /* The float `f` */ TFloat(float f) { f = any(FloatLiteral num).getValue() } or + /* The unicode string `s` */ TUnicode(string s) { // Any string explicitly mentioned in the source code. exists(StrConst str | @@ -90,6 +111,7 @@ newtype TObject = s = "__main__" } or + /* The byte string `s` */ TBytes(string s) { // Any string explicitly mentioned in the source code. exists(StrConst str | @@ -106,6 +128,7 @@ newtype TObject = s = "__main__" } or + /* An instance of `cls`, instantiated at `instantiation` given the `context`. */ TSpecificInstance(ControlFlowNode instantiation, ClassObjectInternal cls, PointsToContext context) { PointsToInternal::pointsTo(instantiation.(CallNode).getFunction(), context, cls, _) and cls.isSpecial() = false and cls.getClassDeclaration().callReturnsInstance() @@ -113,60 +136,74 @@ newtype TObject = literal_instantiation(instantiation, cls, context) } or + /* A non-specific instance `cls` which enters the scope at `def` given the callee `context`. */ TSelfInstance(ParameterDefinition def, PointsToContext context, PythonClassObjectInternal cls) { self_parameter(def, context, cls) } or + /* A bound method */ TBoundMethod(ObjectInternal self, CallableObjectInternal function) { any(ObjectInternal obj).binds(self, _, function) and function.isDescriptor() = true } or + /* Represents any value whose class is known, but nothing else */ TUnknownInstance(BuiltinClassObjectInternal cls) { cls != ObjectInternal::superType() and cls != ObjectInternal::builtin("bool") and cls != ObjectInternal::noneType() } or + /* Represents an instance of `super` */ TSuperInstance(ObjectInternal self, ClassObjectInternal startclass) { super_instantiation(_, self, startclass, _) } or + /* Represents an instance of `classmethod` */ TClassMethod(CallNode instantiation, CallableObjectInternal function) { class_method(instantiation, function, _) } or + /* Represents an instance of `staticmethod` */ TStaticMethod(CallNode instantiation, CallableObjectInternal function) { static_method(instantiation, function, _) } or + /* Represents a builtin tuple */ TBuiltinTuple(Builtin bltn) { bltn.getClass() = Builtin::special("tuple") } or + /* Represents a tuple in the Python source */ TPythonTuple(TupleNode origin, PointsToContext context) { context.appliesTo(origin) } or + /* `type` */ TType() or + /* Represents an instance of `property` */ TProperty(CallNode call, Context ctx, CallableObjectInternal getter) { PointsToInternal::pointsTo(call.getFunction(), ctx, ObjectInternal::property(), _) and PointsToInternal::pointsTo(call.getArg(0), ctx, getter, _) } or + /* Represents a dynamically created class */ TDynamicClass(CallNode instantiation, ClassObjectInternal metacls, PointsToContext context) { PointsToInternal::pointsTo(instantiation.getFunction(), context, metacls, _) and not count(instantiation.getAnArg()) = 1 and Types::getMro(metacls).contains(TType()) } or + /* Represents `sys.version_info`. Acts like a tuple with a range of values depending on the version being analysed. */ TSysVersionInfo() or + /* Represents a module that is inferred to perhaps exist, but is not present in the database. */ TAbsentModule(string name) { missing_imported_module(_, _, name) } or + /* Represents an attribute of a module that is inferred to perhaps exist, but is not present in the database. */ TAbsentModuleAttribute(AbsentModuleObjectInternal mod, string attrname) { ( PointsToInternal::pointsTo(any(AttrNode attr).getObject(attrname), _, mod, _) From e1614886c7f13b6e9d431c8d8980f1f89cc4ece5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 May 2019 17:28:25 +0100 Subject: [PATCH 112/128] Python points-to. Improve API bey renaming a couple of methods and clarifying a few comments. --- .../src/semmle/python/objects/Callables.qll | 16 +++++----- .../ql/src/semmle/python/objects/Classes.qll | 20 ++++++------ .../src/semmle/python/objects/Constants.qll | 16 +++++----- .../src/semmle/python/objects/Descriptors.qll | 12 +++---- .../src/semmle/python/objects/Instances.qll | 16 +++++----- .../ql/src/semmle/python/objects/Modules.qll | 14 ++++---- .../semmle/python/objects/ObjectInternal.qll | 32 +++++++++++-------- .../src/semmle/python/objects/Sequences.qll | 10 +++--- .../src/semmle/python/pointsto/PointsTo.qll | 12 +++---- 9 files changed, 77 insertions(+), 71 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 53671cee4804..74f4624d7ac9 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -67,7 +67,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti result = "Function " + this.getScope().getQualifiedName() } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { this = TPythonFunctionObject(node) and context.appliesTo(node) } @@ -75,7 +75,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti result = TBuiltinClassObject(Builtin::special("FunctionType")) } - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } override Builtin getBuiltin() { none() @@ -182,7 +182,7 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc result = "Builtin-function " + this.getBuiltin().getName() } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } @@ -190,7 +190,7 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc result = TBuiltinClassObject(this.getBuiltin().getClass()) } - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -295,11 +295,11 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod result = TBuiltinClassObject(this.getBuiltin().getClass()) } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -396,11 +396,11 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { result = TBuiltinClassObject(Builtin::special("MethodType")) } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } - override boolean isComparable() { result = false } + override boolean testableForEquality() { result = false } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { this.getFunction().callResult(callee, obj, origin) diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index abe363617adc..69fc1c46b6d0 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -103,7 +103,7 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject result = "class " + this.getScope().getName() } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { this = TPythonClassObject(node) and context.appliesTo(node) } @@ -145,7 +145,7 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject none() } - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } override predicate functionAndOffset(CallableObjectInternal function, int offset) { this.lookup("__init__", function, _) and offset = 1 @@ -164,7 +164,7 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec result = "builtin-class " + this.getBuiltin().getName() } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } @@ -203,7 +203,7 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec none() } - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } } @@ -222,11 +222,11 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { result = this } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } - override boolean isComparable() { result = false } + override boolean testableForEquality() { result = false } override Builtin getBuiltin() { result = Builtin::unknownType() @@ -271,11 +271,11 @@ class TypeInternal extends ClassObjectInternal, TType { result = this } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } override Builtin getBuiltin() { result = Builtin::special("type") @@ -342,7 +342,7 @@ class DynamicallyCreatedClass extends ClassObjectInternal, TDynamicClass { pragma [noinline] override predicate attributesUnknown() { any() } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { this = TDynamicClass(node, _, context) } @@ -350,7 +350,7 @@ class DynamicallyCreatedClass extends ClassObjectInternal, TDynamicClass { none() } - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } override ClassDecl getClassDeclaration() { none() } diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index 4b8d00df0462..fb1bf86dd55f 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -20,7 +20,7 @@ abstract class ConstantObjectInternal extends ObjectInternal { override boolean isClass() { result = false } - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { // Constants aren't callable @@ -98,7 +98,7 @@ private class TrueObjectInternal extends BooleanObjectInternal, TTrue { result = true } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { node.(NameNode).getId() = "True" and context.appliesTo(node) } @@ -122,7 +122,7 @@ private class FalseObjectInternal extends BooleanObjectInternal, TFalse { result = false } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { node.(NameNode).getId() = "False" and context.appliesTo(node) } @@ -150,7 +150,7 @@ private class NoneObjectInternal extends ConstantObjectInternal, TNone { result = TBuiltinClassObject(Builtin::special("NoneType")) } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { node.(NameNode).getId() = "None" and context.appliesTo(node) } @@ -177,7 +177,7 @@ private class IntObjectInternal extends ConstantObjectInternal, TInt { result = "int " + this.intValue().toString() } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { context.appliesTo(node) and node.getNode().(IntegerLiteral).getValue() = this.intValue() } @@ -218,7 +218,7 @@ private class FloatObjectInternal extends ConstantObjectInternal, TFloat { ) } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { context.appliesTo(node) and node.getNode().(FloatLiteral).getValue() = this.floatValue() } @@ -260,7 +260,7 @@ private class UnicodeObjectInternal extends ConstantObjectInternal, TUnicode { result = "'" + this.strValue() + "'" } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { context.appliesTo(node) and node.getNode().(StrConst).getText() = this.strValue() and node.getNode().(StrConst).isUnicode() @@ -301,7 +301,7 @@ private class BytesObjectInternal extends ConstantObjectInternal, TBytes { result = "'" + this.strValue() + "'" } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { context.appliesTo(node) and node.getNode().(StrConst).getText() = this.strValue() and not node.getNode().(StrConst).isUnicode() diff --git a/python/ql/src/semmle/python/objects/Descriptors.qll b/python/ql/src/semmle/python/objects/Descriptors.qll index 4b3295028004..334bd8780619 100644 --- a/python/ql/src/semmle/python/objects/Descriptors.qll +++ b/python/ql/src/semmle/python/objects/Descriptors.qll @@ -37,7 +37,7 @@ class PropertyInternal extends ObjectInternal, TProperty { override boolean booleanValue() { result = true } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { this = TProperty(node, context, _) } @@ -47,7 +47,7 @@ class PropertyInternal extends ObjectInternal, TProperty { override ObjectInternal getClass() { result = ObjectInternal::property() } - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } override Builtin getBuiltin() { none() } @@ -102,7 +102,7 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { override boolean booleanValue() { result = true } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { exists(CallableObjectInternal function | this = TClassMethod(node, function) and class_method(node, function, context) @@ -120,7 +120,7 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { override ObjectInternal getClass() { result = ObjectInternal::classMethod() } - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } override Builtin getBuiltin() { none() } @@ -177,7 +177,7 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { override boolean booleanValue() { result = true } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { exists(CallableObjectInternal function | this = TStaticMethod(node, function) and static_method(node, function, context) @@ -194,7 +194,7 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { override ObjectInternal getClass() { result = ObjectInternal::builtin("staticmethod") } - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } override Builtin getBuiltin() { none() } diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index bfd95aa274eb..ab6cab02f56d 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -63,7 +63,7 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject { result = maybe() } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { this = TSpecificInstance(node, _, context) } @@ -74,7 +74,7 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject { override boolean isClass() { result = false } - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } override ObjectInternal getClass() { this = TSpecificInstance(_, result, _) @@ -166,7 +166,7 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject { result = maybe() } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } @@ -181,7 +181,7 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject { override boolean isClass() { result = false } - override boolean isComparable() { result = false } + override boolean testableForEquality() { result = false } override ObjectInternal getClass() { this = TSelfInstance(_, _, result) @@ -266,7 +266,7 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { result = maybe() } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } @@ -277,7 +277,7 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { override boolean isClass() { result = false } - override boolean isComparable() { result = false } + override boolean testableForEquality() { result = false } override ObjectInternal getClass() { this = TUnknownInstance(result) @@ -375,7 +375,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal { override boolean booleanValue() { result = true } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { exists(ObjectInternal self, ClassObjectInternal startclass | super_instantiation(node, self, startclass, context) and this = TSuperInstance(self, startclass) @@ -400,7 +400,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal { result = ObjectInternal::superType() } - override boolean isComparable() { result = false } + override boolean testableForEquality() { result = false } override Builtin getBuiltin() { none() } diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index d47966238972..99f28a310101 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -28,7 +28,7 @@ abstract class ModuleObjectInternal extends ObjectInternal { override boolean isClass() { result = false } - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } override boolean booleanValue() { result = true @@ -72,7 +72,7 @@ class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleOb result = this.getBuiltin().getName() } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } @@ -129,7 +129,7 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { result = moduleNameFromFile(this.getFolder()) } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } @@ -220,7 +220,7 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule { result = this.getSourceModule().getName() } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } @@ -274,7 +274,7 @@ class AbsentModuleObjectInternal extends ModuleObjectInternal, TAbsentModule { this = TAbsentModule(result) } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { missing_imported_module(node, context, this.getName()) } @@ -328,7 +328,7 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA ) } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { exists(ModuleObjectInternal mod, string name | this = TAbsentModuleAttribute(mod, name) | PointsToInternal::pointsTo(node.(AttrNode).getObject(name), context, mod, _) @@ -375,7 +375,7 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA override boolean isClass() { result = maybe() } - override boolean isComparable() { result = false } + override boolean testableForEquality() { result = false } override boolean booleanValue() { result = maybe() diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index dc1a4ee5c0aa..0e3b2be9ebbe 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -33,7 +33,7 @@ class ObjectInternal extends TObject { * * A bound method would be "introduced" when relevant attribute on an instance * is accessed. In `x = X(); x.m` `x.m` introduces the bound method. */ - abstract predicate introduced(ControlFlowNode node, PointsToContext context); + abstract predicate introducedAt(ControlFlowNode node, PointsToContext context); /** Gets the class declaration for this object, if it is a class with a declaration. */ abstract ClassDecl getClassDeclaration(); @@ -44,10 +44,11 @@ class ObjectInternal extends TObject { /** Gets the class of this object. */ abstract ObjectInternal getClass(); - /** True if this "object" can be meaningfully analysed to determine its boolean value in comparisons. + /** True if this "object" can be meaningfully analysed to determine the boolean value of + * equality tests on it. * For example, `None` or `int` can be, but `int()` or an unknown string cannot. */ - abstract boolean isComparable(); + abstract boolean testableForEquality(); /** Gets the `Builtin` for this object, if any. * Objects (except unknown and undefined values) should attempt to return @@ -56,8 +57,10 @@ class ObjectInternal extends TObject { abstract Builtin getBuiltin(); /** Gets a control flow node that represents the source origin of this - * object. Source code objects should attempt to return - * exactly one result for this method. + * object, if it has a meaningful location in the source code. + * This method exists primarily for providing backwards compatibility and + * locations for source objects. + * Source code objects should attempt to return exactly one result for this method. */ abstract ControlFlowNode getOrigin(); @@ -71,8 +74,9 @@ class ObjectInternal extends TObject { */ abstract predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin); - /** The integer value of things that have integer values. - * That is, ints and bools. + /** The integer value of things that have integer values and whose integer value is + * tracked. + * That is, some ints, mainly small numbers, and bools. */ abstract int intValue(); @@ -84,6 +88,7 @@ class ObjectInternal extends TObject { /** Holds if the function `scope` is called when this object is called and `paramOffset` * is the difference from the parameter position and the argument position. * For a normal function `paramOffset` is 0. For classes and bound-methods it is 1. + * Used by points-to to help determine flow from arguments to parameters. */ abstract predicate calleeAndOffset(Function scope, int paramOffset); @@ -146,6 +151,7 @@ class ObjectInternal extends TObject { /** Holds if the object `function` is called when this object is called and `paramOffset` * is the difference from the parameter position and the argument position. * For a normal function `paramOffset` is 0. For classes and bound-methods it is 1. + * This is used to implement the `CallableValue` public API. */ predicate functionAndOffset(CallableObjectInternal function, int offset) { none() } @@ -183,11 +189,11 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { result = TBuiltinClassObject(this.getBuiltin().getClass()) } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } - override boolean isComparable() { result = false } + override boolean testableForEquality() { result = false } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() @@ -257,11 +263,11 @@ class UnknownInternal extends ObjectInternal, TUnknown { result = TUnknownClass() } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } - override boolean isComparable() { result = false } + override boolean testableForEquality() { result = false } override Builtin getBuiltin() { result = Builtin::unknown() @@ -327,13 +333,13 @@ class UndefinedInternal extends ObjectInternal, TUndefined { override boolean isClass() { result = false } - override boolean isComparable() { result = false } + override boolean testableForEquality() { result = false } override ObjectInternal getClass() { none() } - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index cb686d4b94b6..3f1543785216 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -60,7 +60,7 @@ abstract class TupleObjectInternal extends SequenceObjectInternal { * truth or false in comparisons. For example, `None` or `int` can be, but `int()` * or an unknown string cannot. */ - override boolean isComparable() { result = true } + override boolean testableForEquality() { result = true } /** Holds if `obj` is the result of calling `this` and `origin` is * the origin of `obj`. @@ -94,7 +94,7 @@ abstract class TupleObjectInternal extends SequenceObjectInternal { class BuiltinTupleObjectInternal extends TBuiltinTuple, TupleObjectInternal { - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } @@ -121,7 +121,7 @@ class BuiltinTupleObjectInternal extends TBuiltinTuple, TupleObjectInternal { class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal { - override predicate introduced(ControlFlowNode node, PointsToContext context) { + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { this = TPythonTuple(node, context) } @@ -161,7 +161,7 @@ class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectIntern n = 1 and result = TInt(minor_version()) } - override predicate introduced(ControlFlowNode node, PointsToContext context) { none() } + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() } /** Gets the class declaration for this object, if it is a declared class. */ override ClassDecl getClassDeclaration() { @@ -175,7 +175,7 @@ class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectIntern result.getBuiltin() = this.getClassDeclaration() } - override boolean isComparable() { + override boolean testableForEquality() { result = true } diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index b7b33be3c85f..eed7419fe8d0 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -192,7 +192,7 @@ cached module PointsToInternal { or if_exp_points_to(f, context, value, origin) or - origin = f and value.introduced(f, context) + origin = f and value.introducedAt(f, context) or InterModulePointsTo::import_points_to(f, context, value, origin) or @@ -1266,16 +1266,16 @@ module Expressions { exists(ObjectInternal other, boolean sense | equalityTest(comp, context, operand, opvalue, other, sense) | - opvalue.isComparable() = true and other.isComparable() = true and + opvalue.testableForEquality() = true and other.testableForEquality() = true and ( other = opvalue and result = sense or other != opvalue and result = sense.booleanNot() ) or - opvalue.isComparable() = false and result = maybe() + opvalue.testableForEquality() = false and result = maybe() or - other.isComparable() = false and result = maybe() + other.testableForEquality() = false and result = maybe() ) } @@ -1299,9 +1299,9 @@ module Expressions { or compare(val, other) = 1 and result = sense.booleanNot() or - val.isComparable() = false and result = maybe() + val.testableForEquality() = false and result = maybe() or - other.isComparable() = false and result = maybe() + other.testableForEquality() = false and result = maybe() ) } From 66e64615eed3b350762b87224efabeb3f29ad8e5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 28 May 2019 17:16:50 +0100 Subject: [PATCH 113/128] Python points-to: Improve qldoc and internal API a bit. --- python/ql/src/semmle/python/Flow.qll | 2 +- python/ql/src/semmle/python/objects/Classes.qll | 5 ++++- python/ql/src/semmle/python/objects/Instances.qll | 2 -- python/ql/src/semmle/python/objects/Sequences.qll | 9 ++++----- python/ql/src/semmle/python/pointsto/MRO.qll | 7 +++++-- python/ql/src/semmle/python/pointsto/PointsTo.qll | 15 ++++----------- 6 files changed, 18 insertions(+), 22 deletions(-) diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll index 7e07085666b1..3d6bf79b2b47 100755 --- a/python/ql/src/semmle/python/Flow.qll +++ b/python/ql/src/semmle/python/Flow.qll @@ -1014,7 +1014,7 @@ class BasicBlock extends @py_flow_node { result = this.getLastNode().getAFalseSuccessor().getBasicBlock() } - /** Gets an exceptional successor to this basic block */ + /** Gets an unconditional successor to this basic block */ BasicBlock getAnUnconditionalSuccessor() { result = this.getASuccessor() and not result = this.getATrueSuccessor() and diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 69fc1c46b6d0..6629169f374a 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -15,8 +15,11 @@ abstract class ClassObjectInternal extends ObjectInternal { result = this.getClassDeclaration().getName() } + /** Holds if this is a class whose instances we treat specially, rather than as a generic instance. + * For example, `type` or `int`. + */ boolean isSpecial() { - result = Types::getMro(this).isSpecial() + result = Types::getMro(this).containsSpecial() } /** Looks up the attribute `name` on this class. diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index ab6cab02f56d..527509790990 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -57,7 +57,6 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject { result = this.getOrigin().getNode().toString() } - /** The boolean value of this object, if it has one */ override boolean booleanValue() { //result = this.getClass().instancesBooleanValue() result = maybe() @@ -261,7 +260,6 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { result = "instance of " + this.getClass().(ClassObjectInternal).getName() } - /** The boolean value of this object, if it has one */ override boolean booleanValue() { result = maybe() } diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index 3f1543785216..a66b92059c9c 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -16,8 +16,6 @@ abstract class SequenceObjectInternal extends ObjectInternal { /** Gets the `n`th item of this sequence, if one exists. */ abstract ObjectInternal getItem(int n); - /** The boolean value of this object, this may be both - * true and false if the "object" represents a set of possible objects. */ override boolean booleanValue() { this.length() = 0 and result = false or @@ -37,13 +35,11 @@ abstract class SequenceObjectInternal extends ObjectInternal { abstract class TupleObjectInternal extends SequenceObjectInternal { override string toString() { - this.length() = 0 and result = "()" - or result = "(" + this.contents(0) + ")" } private string contents(int n) { - n = this.length() - 1 and result = this.getItem(n).toString() + n = this.length() and result = "" or result = this.getItem(n).toString() + ", " + this.contents(n+1) } @@ -149,6 +145,9 @@ class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal { } +/** The `sys.version_info` object. We treat this specially to prevent premature pruning and + * false positives when we are unsure of the actual version of Python that the code is expecting. + */ class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectInternal { override string toString() { diff --git a/python/ql/src/semmle/python/pointsto/MRO.qll b/python/ql/src/semmle/python/pointsto/MRO.qll index 7aabc29ebd4e..2d3217cf5563 100644 --- a/python/ql/src/semmle/python/pointsto/MRO.qll +++ b/python/ql/src/semmle/python/pointsto/MRO.qll @@ -231,7 +231,10 @@ class ClassList extends TClassList { reverse_step(this, Empty(), result) } - boolean isSpecial() { + /** Holds if this MRO contains a class whose instances we treat specially, rather than as a generic instance. + * For example, `type` or `int`. + */ + boolean containsSpecial() { this = Empty() and result = false or exists(ClassDecl decl | @@ -239,7 +242,7 @@ class ClassList extends TClassList { if decl.isSpecial() then result = true else - result = this.getTail().isSpecial() + result = this.getTail().containsSpecial() ) } diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index eed7419fe8d0..fb92b96f4b50 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -205,7 +205,10 @@ cached module PointsToInternal { f.(PointsToExtension).pointsTo(context, value, origin) } - /** Holds if the attribute `name` is required for `obj` */ + /** Holds if the attribute `name` is required for `obj` + * For object `x` and attribute `name` it means that there exists somewhere in the code + * `x.name` or `getattr(x, "name")`. + */ cached predicate attributeRequired(ObjectInternal obj, string name) { pointsTo(any(AttrNode a).getObject(name), _, obj, _) or @@ -689,16 +692,6 @@ module InterModulePointsTo { ) } - private predicate importsByImportStar(ModuleObjectInternal mod, ModuleObjectInternal imported) { - exists(ImportStarNode isn | - PointsToInternal::pointsTo(isn.getModule(), _, imported, _) and - isn.getScope() = mod.getSourceModule() - ) - or exists(ModuleObjectInternal mid | - importsByImportStar(mod, mid) and importsByImportStar(mid, imported) - ) - } - predicate ofInterestInExports(ModuleObjectInternal mod, string name) { exists(ImportStarNode imp, ImportStarRefinement def | imp = def.getDefiningNode() and From 3adaf07170c6e912b3e8b1cdcf99c49fdac84ee1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 28 May 2019 17:23:24 +0100 Subject: [PATCH 114/128] Python points-to: Make purpose of '$' variable explicit. --- python/ql/src/semmle/dataflow/SSA.qll | 8 ++++++ .../src/semmle/python/pointsto/PointsTo.qll | 26 ++++++++++--------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/python/ql/src/semmle/dataflow/SSA.qll b/python/ql/src/semmle/dataflow/SSA.qll index 6f8704a10c26..e1ddd73bb267 100755 --- a/python/ql/src/semmle/dataflow/SSA.qll +++ b/python/ql/src/semmle/dataflow/SSA.qll @@ -128,6 +128,14 @@ class EssaVariable extends TEssaDefinition { result = this.getDefinition().getScope() } + /** Holds if this the meta-variable for a scope. + * This is used to attach attributes for undeclared variables implicitly + * defined by `from ... import *` and the like. + */ + predicate isMetaVariable() { + this.getName() = "$" + } + } /* Helper for location_string diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index fb92b96f4b50..c328a63fa897 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -540,7 +540,7 @@ cached module PointsToInternal { /* Undefined variable */ exists(Scope scope | not def.getVariable().getName() = "__name__" and - not def.getVariable().getName() = "$" and + not def.getVariable().isMetaVariable() and def.getScope() = scope and context.appliesToScope(scope) | def.getSourceVariable() instanceof GlobalVariable and scope instanceof Module or @@ -693,13 +693,16 @@ module InterModulePointsTo { } predicate ofInterestInExports(ModuleObjectInternal mod, string name) { - exists(ImportStarNode imp, ImportStarRefinement def | + exists(ImportStarNode imp, ImportStarRefinement def, EssaVariable var | imp = def.getDefiningNode() and - PointsToInternal::pointsTo(imp.getModule(), any(Context ctx | ctx.isImport()), mod, _) | - def.getVariable().getName() = "$" and - ModuleAttributes::attributePointsTo(def.getInput().getDefinition(), name, _, _) - or - def.getVariable().getName() = name and not name = "$" and not name = "*" + PointsToInternal::pointsTo(imp.getModule(), any(Context ctx | ctx.isImport()), mod, _) and + var = def.getVariable() + | + if var.isMetaVariable() then ( + ModuleAttributes::attributePointsTo(def.getInput().getDefinition(), name, _, _) + ) else ( + def.getVariable().getName() = name + ) ) or exists(PackageObjectInternal package | @@ -2082,7 +2085,7 @@ module ModuleAttributes { } EssaVariable moduleStateVariable(ControlFlowNode use) { - result.getName() = "$" and result.getAUse() = use + result.isMetaVariable() and result.getAUse() = use } private EssaVariable moduleStateVarAtExit(Module mod) { @@ -2107,7 +2110,7 @@ module ModuleAttributes { pragma [nomagic] private predicate importStarPointsTo(ImportStarRefinement def, string name, ObjectInternal value, CfgOrigin origin) { - def.getVariable().getName() = "$" and + def.getVariable().isMetaVariable() and exists(ImportStarNode imp, ModuleObjectInternal mod | imp = def.getDefiningNode() and PointsToInternal::pointsTo(imp.getModule(), any(Context ctx | ctx.isImport()), mod, _) @@ -2132,7 +2135,7 @@ module ModuleAttributes { */ pragma [noinline] predicate callsitePointsTo(CallsiteRefinement def, string name, ObjectInternal value, CfgOrigin origin) { - def.getVariable().getName() = "$" and + def.getVariable().isMetaVariable() and exists(EssaVariable var, Function func, PointsToContext callee | InterProceduralPointsTo::callsite_calls_function(def.getCall(), _, func, callee, _) and var = moduleStateVariable(func.getANormalExit()) and @@ -2145,11 +2148,10 @@ module ModuleAttributes { */ pragma [noinline] predicate scopeEntryPointsTo(ScopeEntryDefinition def, string name, ObjectInternal value, CfgOrigin origin) { - def.getVariable().getName() = "$" and + def.getVariable().isMetaVariable() and exists(Module m | def.getScope() = m and not exists(EssaVariable named | named.getName() = name and named.getScope() = m) and - not name = "$" and not name = "*" and value = ObjectInternal::undefined() and origin = CfgOrigin::unknown() | From 77c508f954f2407cf40c15f1ee72eebb95efc5c1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 28 May 2019 17:52:55 +0100 Subject: [PATCH 115/128] Python points-to: More comment clarifications and typo fixes. --- python/ql/src/semmle/python/objects/Classes.qll | 2 +- python/ql/src/semmle/python/objects/Descriptors.qll | 5 +++++ python/ql/src/semmle/python/objects/Modules.qll | 2 ++ python/ql/src/semmle/python/objects/ObjectAPI.qll | 9 ++------- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 6629169f374a..e96b51645762 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -29,7 +29,7 @@ abstract class ClassObjectInternal extends ObjectInternal { * @classmethod * def f(cls): pass * ``` - * `this.lookup("f")` is equivent to `C.__dict__['f']`, which is the class-method + * `this.lookup("f")` is equivalent to `C.__dict__['f']`, which is the class-method * whereas * `this.attr("f") is equivalent to `C.f`, which is a bound-method. */ diff --git a/python/ql/src/semmle/python/objects/Descriptors.qll b/python/ql/src/semmle/python/objects/Descriptors.qll index 334bd8780619..65e6ae9bfc9e 100644 --- a/python/ql/src/semmle/python/objects/Descriptors.qll +++ b/python/ql/src/semmle/python/objects/Descriptors.qll @@ -155,6 +155,11 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { origin = CfgOrigin::unknown() } + /** Holds if attribute lookup on this object may "bind" `cls` to `descriptor`. + * `cls` will always be a class as this is a classmethod. + * Here "bind" means that `instance` is passed to the `classmethod.__get__()` method + * at runtime. The term "bind" is used as this most likely results in a bound-method. + */ pragma [noinline] override predicate binds(ObjectInternal cls, string name, ObjectInternal descriptor) { descriptor = this.getFunction() and exists(ObjectInternal instance | diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index 99f28a310101..e6d30471e69f 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -176,8 +176,10 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject { exists(Module init | init = this.getSourceModule() and ( + /* There is no variable shadowing the name of the child module */ not exists(EssaVariable var | var.getAUse() = init.getANormalExit() and var.getSourceVariable().getName() = name) or + /* The variable shadowing the name of the child module is undefined at exit */ ModuleAttributes::pointsToAtExit(init, name, ObjectInternal::undefined(), _) ) and not name = "__init__" and diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index b2a314cc40af..3a40674dbace 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -47,12 +47,7 @@ class Value extends TObject { /** Gets a call to this object */ CallNode getACall() { - PointsToInternal::pointsTo(result.getFunction(), _, this, _) - or - exists(BoundMethodObjectInternal bm | - PointsToInternal::pointsTo(result.getFunction(), _, bm, _) and - bm.getFunction() = this - ) + result = this.getACall(_) } /** Gets a call to this object with the given `caller` context. */ @@ -249,7 +244,7 @@ class ClassValue extends Value { * @classmethod * def f(cls): pass * ``` - * `this.lookup("f")` is equivent to `C.__dict__['f']`, which is the class-method + * `this.lookup("f")` is equivalent to `C.__dict__['f']`, which is the class-method * whereas * `this.attr("f") is equivalent to `C.f`, which is a bound-method. */ From 7181a025f84db182e0000dde2a2cce918d16c74c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 28 May 2019 21:21:17 +0100 Subject: [PATCH 116/128] Python points-to: Update expected results to account for fixed tuple formatting. --- .../ql/test/library-tests/PointsTo/new/Values.expected | 10 +++++----- .../library-tests/PointsTo/subclass/Checks.expected | 6 +++--- .../library-tests/PointsTo/subclass/Values.expected | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/python/ql/test/library-tests/PointsTo/new/Values.expected b/python/ql/test/library-tests/PointsTo/new/Values.expected index 9e2c8da215b1..650515e70a82 100644 --- a/python/ql/test/library-tests/PointsTo/new/Values.expected +++ b/python/ql/test/library-tests/PointsTo/new/Values.expected @@ -122,7 +122,7 @@ | b_condition.py:96 | ControlFlowNode for y | runtime | None | builtin-class NoneType | | b_condition.py:97 | ControlFlowNode for x | runtime | None | builtin-class NoneType | | b_condition.py:101 | ControlFlowNode for FunctionExpr | import | Function not_or_not | builtin-class function | -| b_condition.py:102 | ControlFlowNode for Tuple | runtime | (builtin-class tuple, builtin-class list) | builtin-class tuple | +| b_condition.py:102 | ControlFlowNode for Tuple | runtime | (builtin-class tuple, builtin-class list, ) | builtin-class tuple | | b_condition.py:102 | ControlFlowNode for UnaryExpr | runtime | bool False | builtin-class bool | | b_condition.py:102 | ControlFlowNode for a | runtime | instance of tuple | builtin-class tuple | | b_condition.py:102 | ControlFlowNode for isinstance | runtime | Builtin-function isinstance | builtin-class builtin_function_or_method | @@ -238,7 +238,7 @@ | h_classes.py:11 | ControlFlowNode for type | import | builtin-class type | builtin-class type | | h_classes.py:11 | ControlFlowNode for type() | import | builtin-class module | builtin-class type | | h_classes.py:12 | ControlFlowNode for Dict | import | Dict | builtin-class dict | -| h_classes.py:12 | ControlFlowNode for Tuple | import | (builtin-class object) | builtin-class tuple | +| h_classes.py:12 | ControlFlowNode for Tuple | import | (builtin-class object, ) | builtin-class tuple | | h_classes.py:12 | ControlFlowNode for object | import | builtin-class object | builtin-class type | | h_classes.py:12 | ControlFlowNode for type | import | builtin-class type | builtin-class type | | h_classes.py:12 | ControlFlowNode for type() | import | type() | builtin-class type | @@ -252,7 +252,7 @@ | h_classes.py:16 | ControlFlowNode for type() | runtime | builtin-class module | builtin-class type | | h_classes.py:17 | ControlFlowNode for type | runtime | builtin-class type | builtin-class type | | h_classes.py:18 | ControlFlowNode for Dict | runtime | Dict | builtin-class dict | -| h_classes.py:18 | ControlFlowNode for Tuple | runtime | (builtin-class object) | builtin-class tuple | +| h_classes.py:18 | ControlFlowNode for Tuple | runtime | (builtin-class object, ) | builtin-class tuple | | h_classes.py:18 | ControlFlowNode for object | runtime | builtin-class object | builtin-class type | | h_classes.py:18 | ControlFlowNode for type | runtime | builtin-class type | builtin-class type | | h_classes.py:18 | ControlFlowNode for type() | runtime | type() | builtin-class type | @@ -830,7 +830,7 @@ | t_type.py:8 | ControlFlowNode for ImportExpr | import | Missing module module | builtin-class module | | t_type.py:9 | ControlFlowNode for type | import | builtin-class type | builtin-class type | | t_type.py:10 | ControlFlowNode for Dict | import | Dict | builtin-class dict | -| t_type.py:10 | ControlFlowNode for Tuple | import | (builtin-class object) | builtin-class tuple | +| t_type.py:10 | ControlFlowNode for Tuple | import | (builtin-class object, ) | builtin-class tuple | | t_type.py:10 | ControlFlowNode for object | import | builtin-class object | builtin-class type | | t_type.py:10 | ControlFlowNode for type | import | builtin-class type | builtin-class type | | t_type.py:10 | ControlFlowNode for type() | import | type() | builtin-class type | @@ -844,7 +844,7 @@ | t_type.py:14 | ControlFlowNode for type() | runtime | builtin-class module | builtin-class type | | t_type.py:15 | ControlFlowNode for type | runtime | builtin-class type | builtin-class type | | t_type.py:16 | ControlFlowNode for Dict | runtime | Dict | builtin-class dict | -| t_type.py:16 | ControlFlowNode for Tuple | runtime | (builtin-class object) | builtin-class tuple | +| t_type.py:16 | ControlFlowNode for Tuple | runtime | (builtin-class object, ) | builtin-class tuple | | t_type.py:16 | ControlFlowNode for object | runtime | builtin-class object | builtin-class type | | t_type.py:16 | ControlFlowNode for type | runtime | builtin-class type | builtin-class type | | t_type.py:16 | ControlFlowNode for type() | runtime | type() | builtin-class type | diff --git a/python/ql/test/library-tests/PointsTo/subclass/Checks.expected b/python/ql/test/library-tests/PointsTo/subclass/Checks.expected index 51fe4db79ad7..0bde3270cd29 100644 --- a/python/ql/test/library-tests/PointsTo/subclass/Checks.expected +++ b/python/ql/test/library-tests/PointsTo/subclass/Checks.expected @@ -1,8 +1,8 @@ | builtin-class dict | builtin-class dict | | builtin-class dict | builtin-class list | -| builtin-class int | (builtin-class float, builtin-class dict) | -| builtin-class int | (builtin-class list, builtin-class int) | -| builtin-class int | (builtin-class list, builtin-class int) | +| builtin-class int | (builtin-class float, builtin-class dict, ) | +| builtin-class int | (builtin-class list, builtin-class int, ) | +| builtin-class int | (builtin-class list, builtin-class int, ) | | builtin-class int | builtin-class dict | | builtin-class int | builtin-class float | | builtin-class int | builtin-class int | diff --git a/python/ql/test/library-tests/PointsTo/subclass/Values.expected b/python/ql/test/library-tests/PointsTo/subclass/Values.expected index 48ec378ee024..86560de46723 100644 --- a/python/ql/test/library-tests/PointsTo/subclass/Values.expected +++ b/python/ql/test/library-tests/PointsTo/subclass/Values.expected @@ -42,13 +42,13 @@ | 13 | ControlFlowNode for isinstance() | bool True | 13 | | 13 | ControlFlowNode for x | () | 9 | | 13 | ControlFlowNode for x | int 0 | 9 | -| 14 | ControlFlowNode for Tuple | (builtin-class list, builtin-class int) | 14 | +| 14 | ControlFlowNode for Tuple | (builtin-class list, builtin-class int, ) | 14 | | 14 | ControlFlowNode for int | builtin-class int | 14 | | 14 | ControlFlowNode for isinstance | Builtin-function isinstance | 14 | | 14 | ControlFlowNode for isinstance() | bool True | 14 | | 14 | ControlFlowNode for list | builtin-class list | 14 | | 14 | ControlFlowNode for x | int 0 | 9 | -| 15 | ControlFlowNode for Tuple | (builtin-class list, builtin-class int) | 15 | +| 15 | ControlFlowNode for Tuple | (builtin-class list, builtin-class int, ) | 15 | | 15 | ControlFlowNode for int | builtin-class int | 15 | | 15 | ControlFlowNode for issubclass | Builtin-function issubclass | 15 | | 15 | ControlFlowNode for issubclass() | bool True | 15 | @@ -56,7 +56,7 @@ | 15 | ControlFlowNode for type | builtin-class type | 15 | | 15 | ControlFlowNode for type() | builtin-class int | 15 | | 15 | ControlFlowNode for x | int 0 | 9 | -| 16 | ControlFlowNode for Tuple | (builtin-class float, builtin-class dict) | 16 | +| 16 | ControlFlowNode for Tuple | (builtin-class float, builtin-class dict, ) | 16 | | 16 | ControlFlowNode for dict | builtin-class dict | 16 | | 16 | ControlFlowNode for float | builtin-class float | 16 | | 16 | ControlFlowNode for issubclass | Builtin-function issubclass | 16 | From 1ed3e5d9bc2a707b02c9703cb09116eb9ec30025 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 29 May 2019 14:33:29 +0100 Subject: [PATCH 117/128] Python points-to: Fix ordering of string constants. --- python/ql/src/semmle/python/pointsto/PointsTo.qll | 4 ++-- .../test/library-tests/PointsTo/comparisons/PointsTo.expected | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index c328a63fa897..6883a17fa44b 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1318,9 +1318,9 @@ module Expressions { or val.strValue() < other.strValue() and result = -1 or - val.strValue() > other.strValue() and result = 0 + val.strValue() > other.strValue() and result = 1 or - val.strValue() = other.strValue() and result = 1 + val.strValue() = other.strValue() and result = 0 } pragma [nomagic] diff --git a/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected b/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected index 29b17d92360b..a89471a901d9 100644 --- a/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected +++ b/python/ql/test/library-tests/PointsTo/comparisons/PointsTo.expected @@ -1,4 +1,3 @@ -| 2 | bool False | | 2 | bool True | | 3 | bool False | | 6 | bool True | From 71ae61c9f760bbd7c72979942a47ceb6c76e1733 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 29 May 2019 14:54:51 +0100 Subject: [PATCH 118/128] Python points-to: Remove duplicate import. --- python/ql/test/library-tests/PointsTo/subclass/Checks.ql | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ql/test/library-tests/PointsTo/subclass/Checks.ql b/python/ql/test/library-tests/PointsTo/subclass/Checks.ql index 68921ec97399..b164406276e5 100644 --- a/python/ql/test/library-tests/PointsTo/subclass/Checks.ql +++ b/python/ql/test/library-tests/PointsTo/subclass/Checks.ql @@ -1,7 +1,6 @@ import python import semmle.python.pointsto.PointsTo -import semmle.python.pointsto.PointsTo from Value sup, Value cls where Expressions::requireSubClass(cls, sup) From 66899942851a850017787f7a04f9b3a2278eb2d7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 30 May 2019 17:17:22 +0100 Subject: [PATCH 119/128] Python points-to: Minor refactoring for clarity. --- .../src/semmle/python/pointsto/PointsTo.qll | 28 +++++++++---------- .../src/semmle/python/types/ClassObject.qll | 2 +- python/ql/src/semmle/python/types/Object.qll | 4 +-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 6883a17fa44b..642cd5e44b83 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1029,7 +1029,7 @@ module InterProceduralPointsTo { ) } - /* Helper for computing ESSA variables at scoepe exit. */ + /* Helper for computing ESSA variables at scope exit. */ private predicate var_at_exit(Variable var, Scope scope, EssaVariable evar) { not var instanceof LocalVariable and evar.getSourceVariable() = var and @@ -1215,31 +1215,31 @@ module Expressions { pragma [noinline] predicate getattr_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, string name) { exists(ControlFlowNode arg1 | - call_to_getattr(call, context, use, arg1) and + call_and_args_for_getattr(call, context, use, arg1) and PointsToInternal::pointsTo(use, context, val, _) and PointsToInternal::pointsToString(arg1, context, name) ) } + pragma[noinline] + private predicate call_and_args_for_getattr(ControlFlowNode call, PointsToContext context, ControlFlowNode arg0, ControlFlowNode arg1) { + exists(ControlFlowNode func | + call2(call, func, arg0, arg1) and + PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("getattr"), _) + ) + } + pragma [noinline] predicate setattr_call(CallNode call, PointsToContext context, ControlFlowNode obj, string name, ObjectInternal val, ControlFlowNode origin) { exists(ControlFlowNode arg1, ControlFlowNode arg2 | - call_to_setattr(call, context, obj, arg1, arg2) and + call_and_args_for_setattr(call, context, obj, arg1, arg2) and PointsToInternal::pointsTo(arg2, context, val, origin) and PointsToInternal::pointsToString(arg1, context, name) ) } pragma[noinline] - private predicate call_to_getattr(ControlFlowNode call, PointsToContext context, ControlFlowNode arg0, ControlFlowNode arg1) { - exists(ControlFlowNode func | - call2(call, func, arg0, arg1) and - PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("getattr"), _) - ) - } - - pragma[noinline] - private predicate call_to_setattr(ControlFlowNode call, PointsToContext context, ControlFlowNode arg0, ControlFlowNode arg1, ControlFlowNode arg2) { + private predicate call_and_args_for_setattr(ControlFlowNode call, PointsToContext context, ControlFlowNode arg0, ControlFlowNode arg1, ControlFlowNode arg2) { exists(ControlFlowNode func | call3(call, func, arg0, arg1, arg2) and PointsToInternal::pointsTo(func, context, ObjectInternal::builtin("setattr"), _) @@ -1247,7 +1247,7 @@ module Expressions { } pragma [noinline] - private boolean otherComparisonEvaluatesTo(CompareNode comp, PointsToContext context, ControlFlowNode operand, ObjectInternal opvalue) { + private boolean containsComparisonEvaluatesTo(CompareNode comp, PointsToContext context, ControlFlowNode operand, ObjectInternal opvalue) { exists(Cmpop op | comp.operands(operand, op, _) or comp.operands(_, op, operand) @@ -1403,7 +1403,7 @@ module Expressions { or result = inequalityEvaluatesTo(expr, context, subexpr, subvalue) or - result = otherComparisonEvaluatesTo(expr, context, subexpr, subvalue) + result = containsComparisonEvaluatesTo(expr, context, subexpr, subvalue) or result = isinstanceEvaluatesTo(expr, context, subexpr, subvalue) or diff --git a/python/ql/src/semmle/python/types/ClassObject.qll b/python/ql/src/semmle/python/types/ClassObject.qll index 84b67400fb45..748b157ee22c 100644 --- a/python/ql/src/semmle/python/types/ClassObject.qll +++ b/python/ql/src/semmle/python/types/ClassObject.qll @@ -78,7 +78,7 @@ class ClassObject extends Object { } /** Whether this class is an old style class. - An old style class is one does not inherit from `object`. */ + An old style class is one that does not inherit from `object`. */ predicate isOldStyle() { Types::isOldStyle(theClass()) } diff --git a/python/ql/src/semmle/python/types/Object.qll b/python/ql/src/semmle/python/types/Object.qll index 26392b0ffde8..14e04ef8c835 100644 --- a/python/ql/src/semmle/python/types/Object.qll +++ b/python/ql/src/semmle/python/types/Object.qll @@ -135,8 +135,8 @@ class Object extends @py_object { * false for None, true for 7 and no result for int(x) */ boolean booleanValue() { - result = booleanFromValue() - and not booleanFromValue() = result.booleanNot() + result = this.booleanFromValue() and + not this.maybe() } final predicate maybe() { From 8e2d6c4fc6638bc88a0a7eca536d1e779c8d3b84 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 30 May 2019 17:32:03 +0100 Subject: [PATCH 120/128] Python points-to: Simplify logic w.r.t. comparisons. --- .../ql/src/semmle/python/objects/Callables.qll | 8 ++++---- python/ql/src/semmle/python/objects/Classes.qll | 10 +++++----- .../ql/src/semmle/python/objects/Constants.qll | 2 +- .../src/semmle/python/objects/Descriptors.qll | 6 +++--- .../ql/src/semmle/python/objects/Instances.qll | 8 ++++---- python/ql/src/semmle/python/objects/Modules.qll | 4 ++-- .../semmle/python/objects/ObjectInternal.qll | 8 ++++---- .../ql/src/semmle/python/objects/Sequences.qll | 6 ++---- .../ql/src/semmle/python/pointsto/PointsTo.qll | 17 +++++++---------- 9 files changed, 32 insertions(+), 37 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 74f4624d7ac9..540115e8c48c 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -75,7 +75,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti result = TBuiltinClassObject(Builtin::special("FunctionType")) } - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } override Builtin getBuiltin() { none() @@ -190,7 +190,7 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc result = TBuiltinClassObject(this.getBuiltin().getClass()) } - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -299,7 +299,7 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod none() } - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() } @@ -400,7 +400,7 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { none() } - override boolean testableForEquality() { result = false } + override predicate notTestableForEquality() { any() } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { this.getFunction().callResult(callee, obj, origin) diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index e96b51645762..4664a2d0b44f 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -148,7 +148,7 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject none() } - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } override predicate functionAndOffset(CallableObjectInternal function, int offset) { this.lookup("__init__", function, _) and offset = 1 @@ -206,7 +206,7 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec none() } - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } } @@ -229,7 +229,7 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass { none() } - override boolean testableForEquality() { result = false } + override predicate notTestableForEquality() { any() } override Builtin getBuiltin() { result = Builtin::unknownType() @@ -278,7 +278,7 @@ class TypeInternal extends ClassObjectInternal, TType { none() } - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } override Builtin getBuiltin() { result = Builtin::special("type") @@ -353,7 +353,7 @@ class DynamicallyCreatedClass extends ClassObjectInternal, TDynamicClass { none() } - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } override ClassDecl getClassDeclaration() { none() } diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index fb1bf86dd55f..1406d9f2b5d9 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -20,7 +20,7 @@ abstract class ConstantObjectInternal extends ObjectInternal { override boolean isClass() { result = false } - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { // Constants aren't callable diff --git a/python/ql/src/semmle/python/objects/Descriptors.qll b/python/ql/src/semmle/python/objects/Descriptors.qll index 65e6ae9bfc9e..a9d6bcb79313 100644 --- a/python/ql/src/semmle/python/objects/Descriptors.qll +++ b/python/ql/src/semmle/python/objects/Descriptors.qll @@ -47,7 +47,7 @@ class PropertyInternal extends ObjectInternal, TProperty { override ObjectInternal getClass() { result = ObjectInternal::property() } - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } override Builtin getBuiltin() { none() } @@ -120,7 +120,7 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { override ObjectInternal getClass() { result = ObjectInternal::classMethod() } - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } override Builtin getBuiltin() { none() } @@ -199,7 +199,7 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { override ObjectInternal getClass() { result = ObjectInternal::builtin("staticmethod") } - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } override Builtin getBuiltin() { none() } diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 527509790990..1a97f54f2a0a 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -73,7 +73,7 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject { override boolean isClass() { result = false } - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } override ObjectInternal getClass() { this = TSpecificInstance(_, result, _) @@ -180,7 +180,7 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject { override boolean isClass() { result = false } - override boolean testableForEquality() { result = false } + override predicate notTestableForEquality() { any() } override ObjectInternal getClass() { this = TSelfInstance(_, _, result) @@ -275,7 +275,7 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { override boolean isClass() { result = false } - override boolean testableForEquality() { result = false } + override predicate notTestableForEquality() { any() } override ObjectInternal getClass() { this = TUnknownInstance(result) @@ -398,7 +398,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal { result = ObjectInternal::superType() } - override boolean testableForEquality() { result = false } + override predicate notTestableForEquality() { any() } override Builtin getBuiltin() { none() } diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index e6d30471e69f..c71bd1112b33 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -28,7 +28,7 @@ abstract class ModuleObjectInternal extends ObjectInternal { override boolean isClass() { result = false } - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } override boolean booleanValue() { result = true @@ -377,7 +377,7 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA override boolean isClass() { result = maybe() } - override boolean testableForEquality() { result = false } + override predicate notTestableForEquality() { any() } override boolean booleanValue() { result = maybe() diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 0e3b2be9ebbe..973d7dca9989 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -48,7 +48,7 @@ class ObjectInternal extends TObject { * equality tests on it. * For example, `None` or `int` can be, but `int()` or an unknown string cannot. */ - abstract boolean testableForEquality(); + abstract predicate notTestableForEquality(); /** Gets the `Builtin` for this object, if any. * Objects (except unknown and undefined values) should attempt to return @@ -193,7 +193,7 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { none() } - override boolean testableForEquality() { result = false } + override predicate notTestableForEquality() { any() } override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() @@ -267,7 +267,7 @@ class UnknownInternal extends ObjectInternal, TUnknown { none() } - override boolean testableForEquality() { result = false } + override predicate notTestableForEquality() { any() } override Builtin getBuiltin() { result = Builtin::unknown() @@ -333,7 +333,7 @@ class UndefinedInternal extends ObjectInternal, TUndefined { override boolean isClass() { result = false } - override boolean testableForEquality() { result = false } + override predicate notTestableForEquality() { any() } override ObjectInternal getClass() { none() diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index a66b92059c9c..17ec3ec6db65 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -56,7 +56,7 @@ abstract class TupleObjectInternal extends SequenceObjectInternal { * truth or false in comparisons. For example, `None` or `int` can be, but `int()` * or an unknown string cannot. */ - override boolean testableForEquality() { result = true } + override predicate notTestableForEquality() { none() } /** Holds if `obj` is the result of calling `this` and `origin` is * the origin of `obj`. @@ -174,9 +174,7 @@ class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectIntern result.getBuiltin() = this.getClassDeclaration() } - override boolean testableForEquality() { - result = true - } + override predicate notTestableForEquality() { none() } /** Gets the `Builtin` for this object, if any. * Objects (except unknown and undefined values) should attempt to return diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 642cd5e44b83..62a969611cc7 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -1262,16 +1262,13 @@ module Expressions { exists(ObjectInternal other, boolean sense | equalityTest(comp, context, operand, opvalue, other, sense) | - opvalue.testableForEquality() = true and other.testableForEquality() = true and - ( - other = opvalue and result = sense - or - other != opvalue and result = sense.booleanNot() - ) + other = opvalue and result = sense + or + other != opvalue and result = sense.booleanNot() or - opvalue.testableForEquality() = false and result = maybe() + opvalue.notTestableForEquality() and result = maybe() or - other.testableForEquality() = false and result = maybe() + other.notTestableForEquality() and result = maybe() ) } @@ -1295,9 +1292,9 @@ module Expressions { or compare(val, other) = 1 and result = sense.booleanNot() or - val.testableForEquality() = false and result = maybe() + val.notTestableForEquality() and result = maybe() or - other.testableForEquality() = false and result = maybe() + other.notTestableForEquality() and result = maybe() ) } From b182abd1199892f50b3b99858f4b62744663ab1b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 May 2019 17:04:22 +0100 Subject: [PATCH 121/128] Python points-to: Fix up a number of overly slow predicates. --- .../src/semmle/python/objects/Instances.qll | 49 ++++--- python/ql/src/semmle/python/pointsto/Base.qll | 6 + .../src/semmle/python/pointsto/PointsTo.qll | 131 ++++++++++-------- 3 files changed, 106 insertions(+), 80 deletions(-) diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index 1a97f54f2a0a..b2e9ea838dfa 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -13,27 +13,36 @@ abstract class InstanceObject extends ObjectInternal { pragma [nomagic] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { - PointsToInternal::attributeRequired(this, name) and - ( - exists(ObjectInternal cls_attr | - this.getClass().(ClassObjectInternal).lookup(name, cls_attr, _) - | - /* If class attribute is not a descriptor, that usually means it is some sort of - * default value and likely overridden by an instance attribute. In that case - * use `unknown` to signal that an attribute exists but to avoid false positives - * f using the default value. - */ - cls_attr.isDescriptor() = false and value = ObjectInternal::unknown() and origin = CfgOrigin::unknown() - or - cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this, value, origin) - ) + exists(ObjectInternal cls_attr | + this.classAttribute(name, cls_attr) + | + /* If class attribute is not a descriptor, that usually means it is some sort of + * default value and likely overridden by an instance attribute. In that case + * use `unknown` to signal that an attribute exists but to avoid false positives + * f using the default value. + */ + cls_attr.isDescriptor() = false and value = ObjectInternal::unknown() and origin = CfgOrigin::unknown() or - exists(EssaVariable self, PythonFunctionObjectInternal init, Context callee | - this.initializer(init, callee) and - self_variable_reaching_init_exit(self) and - self.getScope() = init.getScope() and - AttributePointsTo::variableAttributePointsTo(self, callee, name, value, origin) - ) + cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this, value, origin) + ) + or + this.selfAttribute(name, value, origin) + } + + pragma [noinline] + private predicate classAttribute(string name, ObjectInternal cls_attr) { + PointsToInternal::attributeRequired(this, name) and + this.getClass().(ClassObjectInternal).lookup(name, cls_attr, _) + } + + pragma [noinline] + private predicate selfAttribute(string name, ObjectInternal value, CfgOrigin origin) { + PointsToInternal::attributeRequired(this, name) and + exists(EssaVariable self, PythonFunctionObjectInternal init, Context callee | + this.initializer(init, callee) and + self_variable_reaching_init_exit(self) and + self.getScope() = init.getScope() and + AttributePointsTo::variableAttributePointsTo(self, callee, name, value, origin) ) } diff --git a/python/ql/src/semmle/python/pointsto/Base.qll b/python/ql/src/semmle/python/pointsto/Base.qll index 782163c08426..a596fcf0fb8f 100644 --- a/python/ql/src/semmle/python/pointsto/Base.qll +++ b/python/ql/src/semmle/python/pointsto/Base.qll @@ -417,6 +417,12 @@ class SingleSuccessorGuard extends PyNodeRefinement { ControlFlowNode getTest() { result = this.getDefiningNode() } + + predicate useAndTest(ControlFlowNode use, ControlFlowNode test) { + test = this.getDefiningNode() and + SsaSource::test_refinement(this.getSourceVariable(), use, test) + } + } /** Implicit definition of the names of sub-modules in a package. diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 62a969611cc7..556583d26605 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -417,7 +417,7 @@ cached module PointsToInternal { or attribute_delete_points_to(def, context, value, origin) or - uni_edged_phi_points_to(def, context, value, origin) + uni_edged_pi_points_to(def, context, value, origin) } /** Pass through for `self` for the implicit re-definition of `self` in `self.foo()`. */ @@ -460,16 +460,16 @@ cached module PointsToInternal { result = Conditionals::testEvaluates(def.getTest(), def.getInput().getASourceUse(), context, value, origin) } - /** Holds if ESSA definition, `uniphi`, refers to `(value, origin)`. */ + /** Holds if ESSA definition, `unipi`, refers to `(value, origin)`. */ pragma [noinline] - private predicate uni_edged_phi_points_to(SingleSuccessorGuard uniphi, PointsToContext context, ObjectInternal value, CfgOrigin origin) { - exists(ControlFlowNode test, ControlFlowNode use | + private predicate uni_edged_pi_points_to(SingleSuccessorGuard unipi, PointsToContext context, ObjectInternal value, CfgOrigin origin) { + exists(ControlFlowNode test, ControlFlowNode use, ControlFlowNode orig | /* Because calls such as `len` may create a new variable, we need to go via the source variable * That is perfectly safe as we are only dealing with calls that do not mutate their arguments. */ - use = uniphi.getInput().getASourceUse() and - test = uniphi.getTest() and - uniphi.getSense() = Conditionals::testEvaluates(test, use, context, value, origin.toCfgNode()) + unipi.useAndTest(use, test) and + unipi.getSense() = Conditionals::testEvaluates(test, use, context, value, orig) and + origin = CfgOrigin::fromCfgNode(orig) ) } @@ -785,7 +785,28 @@ module InterProceduralPointsTo { } pragma [noinline] - private predicate call_points_to_simple(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + predicate call_points_to(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { + /* Either not a decorator, or we understand the return value */ + (value != ObjectInternal::unknown() or not f.isDecoratorCall()) and + call_points_to_from_callee(f, context, value, origin) + or + call_result_is_first_argument(f, context) and + PointsToInternal::pointsTo(f.getArg(0), context, value, origin) + or + Expressions::typeCallPointsTo(f, context, value, origin, _, _) + } + + /** Helper for call_points_to to improve join-order */ + private predicate call_result_is_first_argument(CallNode f, PointsToContext context) { + Types::six_add_metaclass(f, context, _, _) + or + /* A decorator and we don't understand it. Use the original, undecorated value */ + f.isDecoratorCall() and call_points_to_from_callee(f, context, ObjectInternal::unknown(), _) + } + + /** Helper for call_points_to to improve join-order */ + pragma [noinline] + private predicate call_points_to_from_callee(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(ObjectInternal func | call(f, context, func) | @@ -806,27 +827,6 @@ module InterProceduralPointsTo { ) } - pragma [noinline] - predicate call_points_to(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - /* Either not a decorator, or we understand the return value */ - (value != ObjectInternal::unknown() or not f.isDecoratorCall()) and - call_points_to_simple(f, context, value, origin) - or - call_result_is_first_argument(f, context) and - PointsToInternal::pointsTo(f.getArg(0), context, value, origin) - or - Expressions::typeCallPointsTo(f, context, value, origin, _, _) - } - - /** Helper for call_points_to to improve join-order */ - private predicate call_result_is_first_argument(CallNode f, PointsToContext context) { - Types::six_add_metaclass(f, context, _, _) - or - /* A decorator and we don't understand it. Use the original, undecorated value */ - f.isDecoratorCall() and call_points_to_simple(f, context, ObjectInternal::unknown(), _) - } - - /** Points-to for parameter. `def foo(param): ...`. */ pragma [noinline] predicate parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { @@ -898,26 +898,27 @@ module InterProceduralPointsTo { /** Helper for parameter_points_to */ pragma [noinline] private predicate special_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { - ( - def.isVarargs() and value = TUnknownInstance(ObjectInternal::builtin("tuple")) - or - def.isKwargs() and value = TUnknownInstance(ObjectInternal::builtin("dict")) - ) - and + special_parameter_value(def, value) and ( context.isRuntime() or - exists(PointsToContext caller, CallNode call, Parameter p | + exists(PointsToContext caller, CallNode call | context.fromCall(call, caller) and context.appliesToScope(def.getScope()) and - p = def.getParameter() and - not exists(call.getArg(p.getPosition())) and - not exists(call.getArgByName(p.getName())) + not exists(call.getArg(def.getParameter().getPosition())) and + not exists(call.getArgByName(def.getParameter().getName())) ) ) and origin = def.getDefiningNode() } + /** Helper predicate for special_parameter_points_to */ + private predicate special_parameter_value(ParameterDefinition p, ObjectInternal value) { + p.isVarargs() and value = TUnknownInstance(ObjectInternal::builtin("tuple")) + or + p.isKwargs() and value = TUnknownInstance(ObjectInternal::builtin("dict")) + } + /** Holds if the `(argument, caller)` pair matches up with `(param, callee)` pair across call. */ cached predicate callsite_argument_transfer(ControlFlowNode argument, PointsToContext caller, ParameterDefinition param, PointsToContext callee) { exists(CallNode call, Function func, int offset | @@ -1178,10 +1179,7 @@ module Expressions { pragma [noinline] predicate typeCallPointsTo(CallNode call, PointsToContext context, ObjectInternal value, ControlFlowNode origin, ControlFlowNode arg, ObjectInternal argvalue) { - not exists(call.getArg(1)) and - arg = call.getArg(0) and - InterProceduralPointsTo::call(call, context, ObjectInternal::builtin("type")) and - PointsToInternal::pointsTo(arg, context, argvalue, _) and + type_call1(call, arg, context, argvalue) and value = argvalue.getClass() and origin = CfgOrigin::fromObject(value).asCfgNodeOrHere(call) } @@ -1467,6 +1465,13 @@ module Expressions { PointsToInternal::pointsTo(use, context, val, _) } + private predicate type_call1(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val) { + PointsToInternal::pointsTo(call.getFunction(), context, ObjectInternal::builtin("type"), _) and + use = call.getArg(0) and + not exists(call.getArg(1)) and + PointsToInternal::pointsTo(use, context, val, _) + } + private predicate hasattr_call(CallNode call, ControlFlowNode use, PointsToContext context, ObjectInternal val, string name) { exists(ControlFlowNode arg1 | call_to_hasattr(call, context, use, arg1) and @@ -1580,11 +1585,7 @@ module Conditionals { test.getAChild*() = use ) or - exists(SingleSuccessorGuard unipi | - unipi.getInput().getASourceUse() = use and - unipi.getTest() = test and - test.getAChild*() = use - ) + any(SingleSuccessorGuard ssg).useAndTest(use, test) } private predicate pinode_test_part(ControlFlowNode outer, ControlFlowNode inner) { @@ -1780,6 +1781,7 @@ cached module Types { ) } + pragma [nomagic] private ClassObjectInternal getInheritedMetaclass(ClassObjectInternal cls) { result = getInheritedMetaclass(cls, 0) or @@ -2026,8 +2028,8 @@ module AttributePointsTo { variableAttributePointsTo(def.getInput(), context, name, value, origin) } - private predicate uniEdgedPhiAttributePointsTo(SingleSuccessorGuard uniphi, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { - variableAttributePointsTo(uniphi.getInput(), context, name, value, origin) + private predicate uniEdgedPhiAttributePointsTo(SingleSuccessorGuard unipi, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { + variableAttributePointsTo(unipi.getInput(), context, name, value, origin) } private predicate piNodeAttributePointsTo(PyEdgeRefinement pi, PointsToContext context, string name, ObjectInternal value, CfgOrigin origin) { @@ -2108,24 +2110,33 @@ module ModuleAttributes { pragma [nomagic] private predicate importStarPointsTo(ImportStarRefinement def, string name, ObjectInternal value, CfgOrigin origin) { def.getVariable().isMetaVariable() and - exists(ImportStarNode imp, ModuleObjectInternal mod | - imp = def.getDefiningNode() and - PointsToInternal::pointsTo(imp.getModule(), any(Context ctx | ctx.isImport()), mod, _) - | + /* Attribute from imported module */ + exists(ModuleObjectInternal mod | + importStarDef(def, _, mod) + and /* Attribute from imported module */ exists(CfgOrigin orig | InterModulePointsTo::moduleExportsBoolean(mod, name) = true and - mod.attribute(name, value, orig) and origin = orig.fix(imp) and - not exists(Variable v | v.getId() = name and v.getScope() = imp.getScope()) + mod.attribute(name, value, orig) and origin = orig.fix(def.getDefiningNode()) and + not exists(Variable v | v.getId() = name and v.getScope() = def.getScope()) ) - or - /* Retain value held before import */ - (InterModulePointsTo::moduleExportsBoolean(mod, name) = false or name.charAt(0) = "_") - and + ) + or + /* Retain value held before import */ + exists(ModuleObjectInternal mod, EssaVariable input | + importStarDef(def, input, mod) and + (InterModulePointsTo::moduleExportsBoolean(mod, name) = false or name.charAt(0) = "_") and attributePointsTo(def.getInput(), name, value, origin) ) } + private predicate importStarDef(ImportStarRefinement def, EssaVariable input, ModuleObjectInternal mod) { + exists(ImportStarNode imp | + def.getVariable().getName() = "$" and imp = def.getDefiningNode() and + input = def.getInput() and PointsToInternal::pointsTo(imp.getModule(), any(Context ctx | ctx.isImport()), mod, _) + ) + } + /** Points-to for a variable (possibly) redefined by a call: * `var = ...; foo(); use(var)` * Where var may be redefined in call to `foo` if `var` escapes (is global or non-local). From 1de0dc9282921f194527fe7ebd1a5b2088049839 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 May 2019 14:17:23 +0100 Subject: [PATCH 122/128] Python taint-tracking: Fix performance of pathalogically slow predicate. --- .../semmle/python/security/TaintTracking.qll | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/python/ql/src/semmle/python/security/TaintTracking.qll b/python/ql/src/semmle/python/security/TaintTracking.qll index ea225f39aac9..e89e09e6fa72 100755 --- a/python/ql/src/semmle/python/security/TaintTracking.qll +++ b/python/ql/src/semmle/python/security/TaintTracking.qll @@ -1209,17 +1209,22 @@ library module TaintFlowImplementation { or exists(string name | param = func.getParameterByName(name) and argument = func.getNamedArgumentForCall(call, name)) or - class_initializer_argument(_, _, call, func, argument, param) + class_initializer_argument(call, func, argument, param) ) ) } + /* Helper for parameter_step */ pragma [noinline] - predicate class_initializer_argument(ClassValue cls, int n, CallNode call, CallableValue func, ControlFlowNode argument, NameNode param) { - call.getFunction().pointsTo(cls) and - cls.lookup("__init__") = func and - call.getArg(n) = argument and - param.getNode() = func.getScope().getArg(n+1) + private predicate class_initializer_argument(CallNode call, CallableValue func, ControlFlowNode argument, NameNode param) { + exists(ClassValue cls | + cls.getACall() = call and + cls.lookup("__init__") = func + ) and + exists(int n | + call.getArg(n) = argument and + param.getNode() = func.getScope().getArg(n+1) + ) } pragma [noinline] From f311c2013e5f9ad813e50b8fa8e4398254a4c710 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 30 May 2019 16:19:43 +0100 Subject: [PATCH 123/128] Python points-to: Cache a few key predicates. --- python/ql/src/semmle/dataflow/SSA.qll | 3 +- .../src/semmle/python/objects/ObjectAPI.qll | 4 +- python/ql/src/semmle/python/pointsto/Base.qll | 3 +- .../src/semmle/python/pointsto/PointsTo.qll | 56 +++++++++---------- .../src/semmle/python/types/ModuleObject.qll | 2 +- 5 files changed, 33 insertions(+), 35 deletions(-) diff --git a/python/ql/src/semmle/dataflow/SSA.qll b/python/ql/src/semmle/dataflow/SSA.qll index e1ddd73bb267..ec81d7856c8c 100755 --- a/python/ql/src/semmle/dataflow/SSA.qll +++ b/python/ql/src/semmle/dataflow/SSA.qll @@ -340,8 +340,7 @@ class PhiFunction extends EssaDefinition, TPhiFunction { } /** Gets the input variable for this phi node on the edge `pred` -> `this.getBasicBlock()`, if any. */ - pragma [noinline] - EssaVariable getInput(BasicBlock pred) { + cached EssaVariable getInput(BasicBlock pred) { result.getDefinition() = this.reachingDefinition(pred) or result.getDefinition() = this.inputEdgeRefinement(pred) diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 3a40674dbace..0be83c1d26ca 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -188,7 +188,7 @@ class CallableValue extends Value { } /** Gets the argument corresponding to the `n'th parameter node of this callable. */ - ControlFlowNode getArgumentForCall(CallNode call, int n) { + cached ControlFlowNode getArgumentForCall(CallNode call, int n) { exists(ObjectInternal called, int offset | PointsToInternal::pointsTo(call.getFunction(), _, called, _) and called.functionAndOffset(this, offset) @@ -204,7 +204,7 @@ class CallableValue extends Value { /** Gets the argument corresponding to the `name`d parameter node of this callable. */ - ControlFlowNode getNamedArgumentForCall(CallNode call, string name) { + cached ControlFlowNode getNamedArgumentForCall(CallNode call, string name) { exists(CallableObjectInternal called, int offset | PointsToInternal::pointsTo(call.getFunction(), _, called, _) and called.functionAndOffset(this, offset) diff --git a/python/ql/src/semmle/python/pointsto/Base.qll b/python/ql/src/semmle/python/pointsto/Base.qll index a596fcf0fb8f..bcc4e18cdd5a 100644 --- a/python/ql/src/semmle/python/pointsto/Base.qll +++ b/python/ql/src/semmle/python/pointsto/Base.qll @@ -589,8 +589,7 @@ module BaseFlow { } /* Helper for this_scope_entry_value_transfer(...). Transfer of values from earlier scope to later on */ - pragma [noinline] - predicate scope_entry_value_transfer_from_earlier(EssaVariable pred_var, Scope pred_scope, ScopeEntryDefinition succ_def, Scope succ_scope) { + cached predicate scope_entry_value_transfer_from_earlier(EssaVariable pred_var, Scope pred_scope, ScopeEntryDefinition succ_def, Scope succ_scope) { exists(SsaSourceVariable var | reaches_exit(pred_var) and pred_var.getScope() = pred_scope and diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 556583d26605..af5133ca3c84 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -124,7 +124,7 @@ module PointsTo { } /* Backwards compatibility */ - deprecated predicate + deprecated cached predicate points_to(ControlFlowNode f, PointsToContext context, Object obj, ClassObject cls, ControlFlowNode origin) { exists(ObjectInternal value | PointsToInternal::pointsTo(f, context, value, origin) and @@ -568,15 +568,34 @@ cached module PointsToInternal { pointsTo(f.getAnOperand(), context, value, origin) } + /* Holds if `import name` will import the module `m`. */ + cached predicate module_imported_as(ModuleObjectInternal m, string name) { + /* Normal imports */ + m.getName() = name + or + /* sys.modules['name'] = m */ + exists(ControlFlowNode sys_modules_flow, ControlFlowNode n, ControlFlowNode mod | + /* Use previous points-to here to avoid slowing down the recursion too much */ + exists(SubscriptNode sub | + sub.getValue() = sys_modules_flow and + pointsTo(sys_modules_flow, _, ObjectInternal::sysModules(), _) and + sub.getIndex() = n and + n.getNode().(StrConst).getText() = name and + sub.(DefinitionNode).getValue() = mod and + pointsTo(mod, _, m, _) + ) + ) + } + } -module InterModulePointsTo { +private module InterModulePointsTo { pragma [noinline] predicate import_points_to(ControlFlowNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { exists(string name, ImportExpr i | i.getAFlowNode() = f and i.getImportedModuleName() = name and - module_imported_as(value, name) and + PointsToInternal::module_imported_as(value, name) and origin = f and context.appliesTo(f) ) @@ -623,25 +642,6 @@ module InterModulePointsTo { ) } - /* Holds if `import name` will import the module `m`. */ - predicate module_imported_as(ModuleObjectInternal m, string name) { - /* Normal imports */ - m.getName() = name - or - /* sys.modules['name'] = m */ - exists(ControlFlowNode sys_modules_flow, ControlFlowNode n, ControlFlowNode mod | - /* Use previous points-to here to avoid slowing down the recursion too much */ - exists(SubscriptNode sub | - sub.getValue() = sys_modules_flow and - PointsToInternal::pointsTo(sys_modules_flow, _, ObjectInternal::sysModules(), _) and - sub.getIndex() = n and - n.getNode().(StrConst).getText() = name and - sub.(DefinitionNode).getValue() = mod and - PointsToInternal::pointsTo(mod, _, m, _) - ) - ) - } - /** Implicit "definition" of the names of submodules at the start of an `__init__.py` file. * * PointsTo isn't exactly how the interpreter works, but is the best approximation we can manage statically. @@ -2077,13 +2077,13 @@ module AttributePointsTo { } -module ModuleAttributes { +cached module ModuleAttributes { private EssaVariable varAtExit(Module mod, string name) { result.getName() = name and result.getAUse() = mod.getANormalExit() } - EssaVariable moduleStateVariable(ControlFlowNode use) { + private EssaVariable moduleStateVariable(ControlFlowNode use) { result.isMetaVariable() and result.getAUse() = use } @@ -2091,7 +2091,7 @@ module ModuleAttributes { result = moduleStateVariable(mod.getANormalExit()) } - predicate pointsToAtExit(Module mod, string name, ObjectInternal value, CfgOrigin origin) { + cached predicate pointsToAtExit(Module mod, string name, ObjectInternal value, CfgOrigin origin) { if exists(varAtExit(mod, name)) then ( PointsToInternal::variablePointsTo(varAtExit(mod, name), any(Context c | c.isImport()), value, origin) ) else ( @@ -2099,7 +2099,7 @@ module ModuleAttributes { ) } - predicate attributePointsTo(EssaVariable var, string name, ObjectInternal value, CfgOrigin origin) { + cached predicate attributePointsTo(EssaVariable var, string name, ObjectInternal value, CfgOrigin origin) { importStarPointsTo(var.getDefinition(), name, value, origin) or callsitePointsTo(var.getDefinition(), name, value, origin) @@ -2142,7 +2142,7 @@ module ModuleAttributes { * Where var may be redefined in call to `foo` if `var` escapes (is global or non-local). */ pragma [noinline] - predicate callsitePointsTo(CallsiteRefinement def, string name, ObjectInternal value, CfgOrigin origin) { + private predicate callsitePointsTo(CallsiteRefinement def, string name, ObjectInternal value, CfgOrigin origin) { def.getVariable().isMetaVariable() and exists(EssaVariable var, Function func, PointsToContext callee | InterProceduralPointsTo::callsite_calls_function(def.getCall(), _, func, callee, _) and @@ -2155,7 +2155,7 @@ module ModuleAttributes { * Since it cannot refer to any actual value, it is set to "undefined" for sub module names. */ pragma [noinline] - predicate scopeEntryPointsTo(ScopeEntryDefinition def, string name, ObjectInternal value, CfgOrigin origin) { + private predicate scopeEntryPointsTo(ScopeEntryDefinition def, string name, ObjectInternal value, CfgOrigin origin) { def.getVariable().isMetaVariable() and exists(Module m | def.getScope() = m and diff --git a/python/ql/src/semmle/python/types/ModuleObject.qll b/python/ql/src/semmle/python/types/ModuleObject.qll index 105f56467adb..6546af0b6dff 100644 --- a/python/ql/src/semmle/python/types/ModuleObject.qll +++ b/python/ql/src/semmle/python/types/ModuleObject.qll @@ -95,7 +95,7 @@ abstract class ModuleObject extends Object { /** Whether this module is imported by 'import name'. For example on a linux system, * the module 'posixpath' is imported as 'os.path' or as 'posixpath' */ predicate importedAs(string name) { - InterModulePointsTo::module_imported_as(theModule(), name) + PointsToInternal::module_imported_as(theModule(), name) } ModuleObject getAnImportedModule() { From f6cc0be4a4ea4315cfbe3362b4115b7047fc8bb5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 May 2019 14:19:52 +0100 Subject: [PATCH 124/128] Python points-to. Move extension to prevent points-to being recomputed. --- .../ql/src/semmle/python/types/Extensions.qll | 28 +++++++++++++++++++ python/ql/src/semmle/python/web/Http.qll | 14 +--------- .../src/semmle/python/web/HttpConstants.qll | 13 +++++++++ .../src/semmle/python/web/bottle/General.qll | 23 --------------- 4 files changed, 42 insertions(+), 36 deletions(-) create mode 100644 python/ql/src/semmle/python/web/HttpConstants.qll diff --git a/python/ql/src/semmle/python/types/Extensions.qll b/python/ql/src/semmle/python/types/Extensions.qll index 8056a52758ba..b4e01e47285f 100644 --- a/python/ql/src/semmle/python/types/Extensions.qll +++ b/python/ql/src/semmle/python/types/Extensions.qll @@ -14,6 +14,7 @@ private import semmle.python.pointsto.PointsTo private import semmle.python.pointsto.PointsToContext private import semmle.python.objects.TObject private import semmle.python.objects.ObjectInternal +private import semmle.python.web.HttpConstants abstract class PointsToExtension extends @py_flow_node { @@ -96,6 +97,33 @@ class RangeIterationVariableFact extends PointsToExtension { } +/* bottle module route constants */ + +class BottleRoutePointToExtension extends PointsToExtension { + + string name; + + BottleRoutePointToExtension() { + exists(DefinitionNode defn | + defn.getScope().(Module).getName() = "bottle" and + this = defn.getValue() and + name = defn.(NameNode).getId() + | + name = "route" or + name = httpVerbLower() + ) + } + + override predicate pointsTo(Context context, ObjectInternal value, ControlFlowNode origin) { + context.isImport() and + exists(CfgOrigin orig | + Module::named("bottle").attr("Bottle").(ClassObjectInternal).attribute(name, value, orig) and + origin = orig.asCfgNodeOrHere(this) + ) + } + +} + /* Python 3.6+ regex module constants */ class ReModulePointToExtension extends PointsToExtension { diff --git a/python/ql/src/semmle/python/web/Http.qll b/python/ql/src/semmle/python/web/Http.qll index ce109258e872..0d8f52f682e5 100644 --- a/python/ql/src/semmle/python/web/Http.qll +++ b/python/ql/src/semmle/python/web/Http.qll @@ -1,25 +1,13 @@ import python import semmle.python.security.TaintTracking import semmle.python.security.strings.External +import HttpConstants /** Generic taint source from a http request */ abstract class HttpRequestTaintSource extends TaintSource { } -/** Gets an http verb */ -string httpVerb() { - result = "GET" or result = "POST" or - result = "PUT" or result = "PATCH" or - result = "DELETE" or result = "OPTIONS" or - result = "HEAD" -} - -/** Gets an http verb, in lower case */ -string httpVerbLower() { - result = httpVerb().toLowerCase() -} - /** Taint kind representing the WSGI environment. * As specified in PEP 3333. https://www.python.org/dev/peps/pep-3333/#environ-variables */ diff --git a/python/ql/src/semmle/python/web/HttpConstants.qll b/python/ql/src/semmle/python/web/HttpConstants.qll new file mode 100644 index 000000000000..a20b89f9f630 --- /dev/null +++ b/python/ql/src/semmle/python/web/HttpConstants.qll @@ -0,0 +1,13 @@ + +/** Gets an http verb */ +string httpVerb() { + result = "GET" or result = "POST" or + result = "PUT" or result = "PATCH" or + result = "DELETE" or result = "OPTIONS" or + result = "HEAD" +} + +/** Gets an http verb, in lower case */ +string httpVerbLower() { + result = httpVerb().toLowerCase() +} diff --git a/python/ql/src/semmle/python/web/bottle/General.qll b/python/ql/src/semmle/python/web/bottle/General.qll index 7f381d70f226..e7751ac5d152 100644 --- a/python/ql/src/semmle/python/web/bottle/General.qll +++ b/python/ql/src/semmle/python/web/bottle/General.qll @@ -54,26 +54,3 @@ class BottleRoute extends ControlFlowNode { } -/* bottle module route constants */ - -class BottleRoutePointToExtension extends CustomPointsToFact { - - string name; - - BottleRoutePointToExtension() { - exists(DefinitionNode defn | - defn.getScope().(Module).getName() = "bottle" and - this = defn.getValue() and - name = defn.(NameNode).getId() - | - name = "route" or - name = httpVerbLower() - ) - } - - override predicate pointsTo(Context context, Object value, ClassObject cls, ControlFlowNode origin) { - context.isImport() and - ModuleObject::named("bottle").attr("Bottle").(ClassObject).attributeRefersTo(name, value, cls, origin) - } -} - From 66ba1079f35932e7144a3b2333f66113043bbb46 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 May 2019 14:56:11 +0100 Subject: [PATCH 125/128] Python points-to: Cache objects. --- python/ql/src/semmle/python/objects/TObject.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 3a60947a9f1d..c6e398e5691e 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -7,7 +7,7 @@ private import semmle.python.pointsto.PointsToContext /** Internal type backing `ObjectInternal` and `Value` * See `ObjectInternal.qll` for an explanation of the API. */ -newtype TObject = +cached newtype TObject = /* Builtin class objects */ TBuiltinClassObject(Builtin bltn) { bltn.isClass() and From 8d4a8a6c6b595697d72315b02cc99f19be55bc24 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 May 2019 15:31:29 +0100 Subject: [PATCH 126/128] Python: Reduce the number of strings and ints identified and possible hard-coded credentials. --- .../Security/CWE-798/HardcodedCredentials.ql | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/python/ql/src/Security/CWE-798/HardcodedCredentials.ql b/python/ql/src/Security/CWE-798/HardcodedCredentials.ql index daaa9d83bb82..4c23899a12e9 100644 --- a/python/ql/src/Security/CWE-798/HardcodedCredentials.ql +++ b/python/ql/src/Security/CWE-798/HardcodedCredentials.ql @@ -53,6 +53,10 @@ predicate capitalized_word(StrConst str) { str.getText().regexpMatch("[A-Z][a-z]+") } +predicate format_string(StrConst str) { + str.getText().matches("%{%}%") +} + predicate maybeCredential(ControlFlowNode f) { /* A string that is not too short and unlikely to be text or an identifier. */ exists(StrConst str | @@ -66,20 +70,21 @@ predicate maybeCredential(ControlFlowNode f) { /* Not too repetitive */ exists(int chars | chars = char_count(str) | - chars > 20 or - chars > str.getText().length()/2 + chars > 15 or + chars*3 > str.getText().length()*2 ) and not possible_reflective_name(str.getText()) and - not capitalized_word(str) + not capitalized_word(str) and + not format_string(str) ) or - /* Or, an integer with at least 8 digits */ + /* Or, an integer with over 32 bits */ exists(IntegerLiteral lit | f.getNode() = lit | - not exists(lit.getValue()) - or - lit.getValue() > 10000000 + not exists(lit.getValue()) and + /* Not a set of flags or round number */ + not lit.getN().matches("%00%") ) } From 6a79e0aaa7b934ff9dc218b10f1f72f7605b6dc3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 May 2019 15:51:37 +0100 Subject: [PATCH 127/128] Python points-to: Prevent bad magic on 'refersTo' member predicates. --- python/ql/src/semmle/python/Exprs.qll | 20 +++----------------- python/ql/src/semmle/python/Flow.qll | 3 +++ 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/python/ql/src/semmle/python/Exprs.qll b/python/ql/src/semmle/python/Exprs.qll index 2dc201f7bf42..82c5bfd52191 100644 --- a/python/ql/src/semmle/python/Exprs.qll +++ b/python/ql/src/semmle/python/Exprs.qll @@ -85,30 +85,16 @@ class Expr extends Expr_, AstNode { /** Gets what this expression might "refer-to" in the given `context`. */ predicate refersTo(Context context, Object obj, ClassObject cls, AstNode origin) { - exists(Value value, ControlFlowNode cfgorigin | - PointsTo::pointsTo(this.getAFlowNode(), context, value, cfgorigin) and - origin.getAFlowNode() = cfgorigin and - cls = value.getClass().getSource() | - if exists(value.getSource()) then - obj = value.getSource() - else - obj = cfgorigin - ) + this.getAFlowNode().refersTo(context, obj, cls, origin.getAFlowNode()) } /** Whether this expression might "refer-to" to `value` which is from `origin` * Unlike `this.refersTo(value, _, origin)`, this predicate includes results * where the class cannot be inferred. */ + pragma[nomagic] predicate refersTo(Object obj, AstNode origin) { - exists(Value value, ControlFlowNode cfgorigin | - PointsTo::pointsTo(this.getAFlowNode(), _, value, cfgorigin) and - origin.getAFlowNode() = cfgorigin and - if exists(value.getSource()) then - obj = value.getSource() - else - obj = cfgorigin - ) + this.getAFlowNode().refersTo(obj, origin.getAFlowNode()) } /** Equivalent to `this.refersTo(value, _)` */ diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll index 3d6bf79b2b47..d2cb71fd5474 100755 --- a/python/ql/src/semmle/python/Flow.qll +++ b/python/ql/src/semmle/python/Flow.qll @@ -237,12 +237,14 @@ class ControlFlowNode extends @py_flow_node { * precise, but may not provide information for a significant number of flow-nodes. * If the class is unimportant then use `refersTo(value)` or `refersTo(value, origin)` instead. */ + pragma [nomagic] predicate refersTo(Object obj, ClassObject cls, ControlFlowNode origin) { this.refersTo(_, obj, cls, origin) } /** Gets what this expression might "refer-to" in the given `context`. */ + pragma [nomagic] predicate refersTo(Context context, Object obj, ClassObject cls, ControlFlowNode origin) { not obj = unknownValue() and not cls = theUnknownType() and @@ -253,6 +255,7 @@ class ControlFlowNode extends @py_flow_node { * Unlike `this.refersTo(value, _, origin)` this predicate includes results * where the class cannot be inferred. */ + pragma [nomagic] predicate refersTo(Object obj, ControlFlowNode origin) { not obj = unknownValue() and PointsTo::points_to(this, _, obj, _, origin) From fc2ac891f8674120f3d8e96074d193142e91a86a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Jun 2019 15:53:36 +0100 Subject: [PATCH 128/128] Python taint-tracking: Don't track strings through json.decode(). --- python/ql/src/semmle/python/security/strings/Basic.qll | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ql/src/semmle/python/security/strings/Basic.qll b/python/ql/src/semmle/python/security/strings/Basic.qll index 5e92543ec086..723616576767 100755 --- a/python/ql/src/semmle/python/security/strings/Basic.qll +++ b/python/ql/src/semmle/python/security/strings/Basic.qll @@ -81,6 +81,7 @@ private predicate str_format(ControlFlowNode fromnode, CallNode tonode) { /* tonode = codec.[en|de]code(fromnode)*/ private predicate encode_decode(ControlFlowNode fromnode, CallNode tonode) { exists(FunctionObject func, string name | + not func.getFunction().isMethod() and func.getACall() = tonode and tonode.getAnArg() = fromnode and func.getName() = name |