From e3932dfeb5df7ac4435136457771f8a9301321e4 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 26 Sep 2019 14:52:56 -0700 Subject: [PATCH 1/4] Add test --- .../Analyzer/Handlers/FromImportHandler.cs | 1 - src/Analysis/Ast/Test/ReferencesTests.cs | 27 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index b972b0023..8484e9e5f 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Modules; diff --git a/src/Analysis/Ast/Test/ReferencesTests.cs b/src/Analysis/Ast/Test/ReferencesTests.cs index bf1fa2418..64e0a99a5 100644 --- a/src/Analysis/Ast/Test/ReferencesTests.cs +++ b/src/Analysis/Ast/Test/ReferencesTests.cs @@ -441,5 +441,32 @@ public async Task ExtendAllAssignment() { all.References[3].Span.Should().Be(12, 1, 12, 8); all.References[4].Span.Should().Be(13, 1, 13, 8); } + + [TestMethod, Priority(0)] + public async Task VariableInCallParameters() { + const string code = @" +from constants import * +import constants + +print(VARIABLE1) +print(constants.VARIABLE2) + +print(VARIABLE2) +print(constants.VARIABLE1) +"; + await TestData.CreateTestSpecificFileAsync("constants.py", @" +VARIABLE1 = 'afad' +VARIABLE2 = 'dcef' +"); + var analysis = await GetAnalysisAsync(code); + var all = analysis.Should().HaveVariable("VARIABLE1").Which; + all.Definition.Span.Should().Be(9, 1, 9, 8); + all.References.Should().HaveCount(5); + all.References[0].Span.Should().Be(9, 1, 9, 8); + all.References[1].Span.Should().Be(10, 1, 10, 8); + all.References[2].Span.Should().Be(11, 1, 11, 8); + all.References[3].Span.Should().Be(12, 1, 12, 8); + all.References[4].Span.Should().Be(13, 1, 13, 8); + } } } From d611e4af03f31e1fe81dc61f47a46a3f8a8b82f3 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 26 Sep 2019 15:33:26 -0700 Subject: [PATCH 2/4] Handle references in function call arguments --- .../Ast/Impl/Analyzer/AnalysisWalker.cs | 8 ++--- .../Evaluation/ExpressionEval.Callables.cs | 18 ++++++++++ .../Analyzer/Handlers/FromImportHandler.cs | 15 ++++++-- src/Analysis/Ast/Test/ReferencesTests.cs | 34 +++++++++++-------- src/Parsing/Impl/Ast/CallExpression.cs | 1 - 5 files changed, 51 insertions(+), 25 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs index 8f5743c70..285fc2f86 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs @@ -69,12 +69,8 @@ public override bool Walk(ExpressionStatement node) { case Comprehension comp: Eval.ProcessComprehension(comp); return false; - case CallExpression callex when callex.Target is NameExpression nex && !string.IsNullOrEmpty(nex.Name): - Eval.LookupNameInScopes(nex.Name)?.AddReference(Eval.GetLocationOfName(nex)); - return true; - case CallExpression callex when callex.Target is MemberExpression mex && !string.IsNullOrEmpty(mex.Name): - var t = Eval.GetValueFromExpression(mex.Target)?.GetPythonType(); - t?.GetMember(mex.Name)?.AddReference(Eval.GetLocationOfName(mex)); + case CallExpression callex: + Eval.ProcessCallForReferences(callex); return true; default: return base.Walk(node); diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index 098595dde..c3b512fac 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -383,5 +383,23 @@ private void DeclareParameter(Parameter p, ParameterInfo pi) { DeclareVariable(p.Name, paramType.CreateInstance(ArgumentSet.Empty(p.NameExpression, this)), VariableSource.Declaration, p.NameExpression); } + + internal void ProcessCallForReferences(CallExpression callExpr) { + switch (callExpr.Target) { + case NameExpression nex when !string.IsNullOrEmpty(nex.Name): + // Add reference to the function + this.LookupNameInScopes(nex.Name)?.AddReference(GetLocationOfName(nex)); + break; + case MemberExpression mex when !string.IsNullOrEmpty(mex.Name): { + var t = GetValueFromExpression(mex.Target)?.GetPythonType(); + t?.GetMember(mex.Name)?.AddReference(GetLocationOfName(mex)); + break; + } + } + // Add references to all arguments. + foreach (var arg in callExpr.Args) { + GetValueFromExpression(arg.Expression); + } + } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index f153cae4b..3f6ec0113 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -99,20 +99,29 @@ private void HandleModuleImportStar(PythonVariableModule variableModule, IImport /// Import search result. /// Name of the variable to declare, such as 'd' in 'from a.b import c as d'. /// Position of the import statement. - /// Name expression of the variable. - private void DeclareVariable(PythonVariableModule variableModule, string memberName, IImportSearchResult imports, string variableName, int importPosition, Node nameExpression) { + /// Location of the variable name expression. + private void DeclareVariable(PythonVariableModule variableModule, string memberName, IImportSearchResult imports, string variableName, int importPosition, Node nameLocation) { // First try imports since child modules should win, i.e. in 'from a.b import c' // 'c' should be a submodule if 'b' has one, even if 'b' also declares 'c = 1'. var value = GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName); + // First try exported or child submodules. value = value ?? variableModule.GetMember(memberName); + // Value may be variable or submodule. If it is variable, we need it in order to add reference. var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName]; value = variable?.Value?.Equals(value) == true ? variable : value; + // If nothing is exported, variables are still accessible. value = value ?? variableModule.Analysis?.GlobalScope?.Variables[memberName]?.Value ?? Eval.UnknownType; + // Do not allow imported variables to override local declarations - Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression, CanOverwriteVariable(variableName, importPosition)); + var canOverwrite = CanOverwriteVariable(variableName, importPosition); + + // Do not declare references to '*' + var locationExpression = nameLocation is NameExpression nex && nex.Name == "*" ? null : nameLocation; + Eval.DeclareVariable(variableName, value, VariableSource.Import, locationExpression, canOverwrite); + // Make sure module is loaded and analyzed. if (value is IPythonModule m) { ModuleResolution.GetOrLoadModule(m.Name); diff --git a/src/Analysis/Ast/Test/ReferencesTests.cs b/src/Analysis/Ast/Test/ReferencesTests.cs index 64e0a99a5..1d32dd5e1 100644 --- a/src/Analysis/Ast/Test/ReferencesTests.cs +++ b/src/Analysis/Ast/Test/ReferencesTests.cs @@ -449,24 +449,28 @@ from constants import * import constants print(VARIABLE1) -print(constants.VARIABLE2) - -print(VARIABLE2) print(constants.VARIABLE1) +x = print(VARIABLE1) "; - await TestData.CreateTestSpecificFileAsync("constants.py", @" -VARIABLE1 = 'afad' -VARIABLE2 = 'dcef' -"); + await TestData.CreateTestSpecificFileAsync("constants.py", @"VARIABLE1 = 'afad'"); var analysis = await GetAnalysisAsync(code); - var all = analysis.Should().HaveVariable("VARIABLE1").Which; - all.Definition.Span.Should().Be(9, 1, 9, 8); - all.References.Should().HaveCount(5); - all.References[0].Span.Should().Be(9, 1, 9, 8); - all.References[1].Span.Should().Be(10, 1, 10, 8); - all.References[2].Span.Should().Be(11, 1, 11, 8); - all.References[3].Span.Should().Be(12, 1, 12, 8); - all.References[4].Span.Should().Be(13, 1, 13, 8); + var v1 = analysis.Should().HaveVariable("VARIABLE1").Which; + + v1.Definition.Span.Should().Be(2, 1, 2, 10); + v1.Definition.DocumentUri.AbsolutePath.Should().Contain("constants.py"); + + v1.References.Should().HaveCount(4); + v1.References[0].Span.Should().Be(2, 1, 2, 10); + v1.References[0].DocumentUri.AbsolutePath.Should().Contain("constants.py"); + + v1.References[1].Span.Should().Be(5, 7, 5, 16); + v1.References[1].DocumentUri.AbsolutePath.Should().Contain("module.py"); + + v1.References[2].Span.Should().Be(6, 17, 6, 26); + v1.References[2].DocumentUri.AbsolutePath.Should().Contain("module.py"); + + v1.References[3].Span.Should().Be(7, 11, 7, 20); + v1.References[3].DocumentUri.AbsolutePath.Should().Contain("module.py"); } } } diff --git a/src/Parsing/Impl/Ast/CallExpression.cs b/src/Parsing/Impl/Ast/CallExpression.cs index 9a5231862..00522512f 100644 --- a/src/Parsing/Impl/Ast/CallExpression.cs +++ b/src/Parsing/Impl/Ast/CallExpression.cs @@ -17,7 +17,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Parsing.Ast { From 036a809cdafca637a12b71b4e906478ae2875950 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 26 Sep 2019 15:39:03 -0700 Subject: [PATCH 3/4] Baseline --- src/Analysis/Ast/Test/ReferencesTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Ast/Test/ReferencesTests.cs b/src/Analysis/Ast/Test/ReferencesTests.cs index 1d32dd5e1..03a83d6f4 100644 --- a/src/Analysis/Ast/Test/ReferencesTests.cs +++ b/src/Analysis/Ast/Test/ReferencesTests.cs @@ -456,11 +456,11 @@ import constants var analysis = await GetAnalysisAsync(code); var v1 = analysis.Should().HaveVariable("VARIABLE1").Which; - v1.Definition.Span.Should().Be(2, 1, 2, 10); + v1.Definition.Span.Should().Be(1, 1, 1, 10); v1.Definition.DocumentUri.AbsolutePath.Should().Contain("constants.py"); v1.References.Should().HaveCount(4); - v1.References[0].Span.Should().Be(2, 1, 2, 10); + v1.References[0].Span.Should().Be(1, 1, 1, 10); v1.References[0].DocumentUri.AbsolutePath.Should().Contain("constants.py"); v1.References[1].Span.Should().Be(5, 7, 5, 16); From 4f22f0bc5950a1f715600ba7bc5f82d989cb46bc Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 26 Sep 2019 16:07:20 -0700 Subject: [PATCH 4/4] Limit to user code, allow builtin functions --- .../Evaluation/ExpressionEval.Callables.cs | 5 ++++ src/Analysis/Ast/Impl/Types/LocatedMember.cs | 6 ++++- src/Analysis/Ast/Impl/Types/PythonType.cs | 8 ------- src/Analysis/Ast/Test/ReferencesTests.cs | 23 +++++++++++++++++++ 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index c3b512fac..462e2fee0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -385,6 +385,10 @@ private void DeclareParameter(Parameter p, ParameterInfo pi) { } internal void ProcessCallForReferences(CallExpression callExpr) { + if (Module.ModuleType != ModuleType.User) { + return; + } + switch (callExpr.Target) { case NameExpression nex when !string.IsNullOrEmpty(nex.Name): // Add reference to the function @@ -396,6 +400,7 @@ internal void ProcessCallForReferences(CallExpression callExpr) { break; } } + // Add references to all arguments. foreach (var arg in callExpr.Args) { GetValueFromExpression(arg.Expression); diff --git a/src/Analysis/Ast/Impl/Types/LocatedMember.cs b/src/Analysis/Ast/Impl/Types/LocatedMember.cs index 2a70b4004..ca863d95b 100644 --- a/src/Analysis/Ast/Impl/Types/LocatedMember.cs +++ b/src/Analysis/Ast/Impl/Types/LocatedMember.cs @@ -54,7 +54,11 @@ public virtual IReadOnlyList References { public virtual void AddReference(Location location) { lock (this) { - if(this.DeclaringModule == null || this.DeclaringModule?.ModuleType == ModuleType.Builtins) { + // In order to limit memory consumption we normally don't track references + // to builtin types such as int or list. Exception is functions like 'print' + // since it user may want to find all references to them. + if (this.DeclaringModule == null || + (this.DeclaringModule?.ModuleType == ModuleType.Builtins && MemberType != PythonMemberType.Function)) { return; } // Don't add references to library code. diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index 4abb24c38..53adbe358 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -47,14 +47,6 @@ private PythonType(string name, Location location, BuiltinTypeId typeId) : base( #region ILocatedMember public override PythonMemberType MemberType => _typeId.GetMemberId(); - - public override void AddReference(Location location) { - if (DeclaringModule == null || DeclaringModule.ModuleType == ModuleType.Builtins) { - return; - } - - base.AddReference(location); - } #endregion #region IPythonType diff --git a/src/Analysis/Ast/Test/ReferencesTests.cs b/src/Analysis/Ast/Test/ReferencesTests.cs index 03a83d6f4..a27a26754 100644 --- a/src/Analysis/Ast/Test/ReferencesTests.cs +++ b/src/Analysis/Ast/Test/ReferencesTests.cs @@ -472,5 +472,28 @@ import constants v1.References[3].Span.Should().Be(7, 11, 7, 20); v1.References[3].DocumentUri.AbsolutePath.Should().Contain("module.py"); } + + [TestMethod, Priority(0)] + public async Task LibraryFunction() { + const string code = @" +print(1) +print(2) +"; + var analysis = await GetAnalysisAsync(code); + var b = analysis.Document.Interpreter.ModuleResolution.BuiltinsModule; + var print = b.Analysis.Should().HaveVariable("print").Which; + + print.Definition.Span.Should().Be(1, 1, 1, 1); + + print.References.Should().HaveCount(3); + print.References[0].Span.Should().Be(1, 1, 1, 1); + print.References[0].DocumentUri.AbsolutePath.Should().Contain("python.pyi"); + + print.References[1].Span.Should().Be(2, 1, 2, 6); + print.References[1].DocumentUri.AbsolutePath.Should().Contain("module.py"); + + print.References[2].Span.Should().Be(3, 1, 3, 6); + print.References[2].DocumentUri.AbsolutePath.Should().Contain("module.py"); + } } }