diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs index d62f11fe2..944c6a9e9 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs @@ -34,7 +34,7 @@ private enum KeyType { Default, Typeshed, LibraryAsDocument } public AnalysisModuleKey(IPythonModule module) { Name = module.Name; FilePath = module.ModuleType == ModuleType.CompiledBuiltin ? null : module.FilePath; - _type = module is StubPythonModule stub && stub.IsTypeshed + _type = module is StubPythonModule stub && stub.IsTypeshed ? KeyType.Typeshed : module.ModuleType == ModuleType.Library && module is IDocument document && document.IsOpen ? KeyType.LibraryAsDocument diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs index 2179e1565..844d46a26 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs @@ -20,7 +20,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Core.DependencyResolution; -using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; @@ -264,8 +263,8 @@ private HashSet FindDependencies(IPythonModule module, Python return dependencies; } - private static bool Ignore(IModuleManagement moduleResolution, string name) - => moduleResolution.BuiltinModuleName.EqualsOrdinal(name) || moduleResolution.GetSpecializedModule(name) != null; + private static bool Ignore(IModuleManagement moduleResolution, string fullName, string modulePath) + => moduleResolution.BuiltinModuleName.EqualsOrdinal(fullName) || moduleResolution.GetSpecializedModule(fullName, modulePath) != null; private void UpdateAnalysisTcs(int analysisVersion) { _analysisVersion = analysisVersion; @@ -332,10 +331,10 @@ public override bool Walk(FromImportStatement fromImport) { private void HandleSearchResults(IImportSearchResult searchResult) { switch (searchResult) { - case ModuleImport moduleImport when !Ignore(_moduleResolution, moduleImport.FullName): + case ModuleImport moduleImport when !Ignore(_moduleResolution, moduleImport.FullName, moduleImport.ModulePath): Dependencies.Add(new AnalysisModuleKey(moduleImport.FullName, moduleImport.ModulePath, _isTypeshed)); return; - case PossibleModuleImport possibleModuleImport when !Ignore(_moduleResolution, possibleModuleImport.PrecedingModuleFullName): + case PossibleModuleImport possibleModuleImport when !Ignore(_moduleResolution, possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath): Dependencies.Add(new AnalysisModuleKey(possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath, _isTypeshed)); return; default: diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 919880435..753a3ef92 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -47,6 +47,7 @@ internal sealed class PythonAnalyzerSession { private readonly IPythonAnalyzer _analyzer; private readonly ILogger _log; private readonly ITelemetryService _telemetry; + private readonly IModuleDatabaseService _moduleDatabaseService; private State _state; private bool _isCanceled; @@ -86,6 +87,7 @@ public PythonAnalyzerSession(IServiceManager services, _analyzer = _services.GetService(); _log = _services.GetService(); _telemetry = _services.GetService(); + _moduleDatabaseService = _services.GetService(); _progress = progress; } @@ -297,7 +299,6 @@ private void AnalyzeEntry() { } var startTime = stopWatch?.Elapsed ?? TimeSpan.Zero; - AnalyzeEntry(null, _entry, module, ast, Version); LogCompleted(null, module, stopWatch, startTime); @@ -383,8 +384,12 @@ private IDocumentAnalysis CreateAnalysis(IDependencyChainNode x is ImportStatement || x is FromImportStatement); document.SetAst(ast); + var eval = new ExpressionEval(walker.Eval.Services, document, ast); - return new LibraryAnalysis(document, version, walker.GlobalScope, eval, walker.StarImportMemberNames); + var analysis = new LibraryAnalysis(document, version, walker.GlobalScope, eval, walker.StarImportMemberNames); + + _moduleDatabaseService?.StoreModuleAnalysisAsync(analysis, CancellationToken.None).DoNotWait(); + return analysis; } private enum State { diff --git a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs index fa8c46d9d..aa82e938a 100644 --- a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs @@ -26,7 +26,6 @@ public static LocationInfo GetLocation(this Node node, IExpressionEvaluator eval return GetLocation(node, eval.Ast, eval.Module); } - public static LocationInfo GetLocation(this Node node, IDocumentAnalysis analysis) { if (node == null || node.StartIndex >= node.EndIndex) { return LocationInfo.Empty; diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs index 89d31d696..ea729b030 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs @@ -56,7 +56,7 @@ public interface IModuleManagement: IModuleResolution { /// /// Returns specialized module, if any. /// - IPythonModule GetSpecializedModule(string name); + IPythonModule GetSpecializedModule(string name, string modulePath = null); /// /// Root directory of the path resolver. diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 366fe8f46..67262f3a9 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -79,6 +79,7 @@ protected PythonModule(string name, ModuleType moduleType, IServiceContainer ser Log = services.GetService(); Interpreter = services.GetService(); Analysis = new EmptyAnalysis(services, this); + GlobalScope = Analysis.GlobalScope; _diagnosticsService = services.GetService(); SetDeclaringModule(this); @@ -150,10 +151,10 @@ public virtual string Documentation { #endregion #region IMemberContainer - public virtual IMember GetMember(string name) => Analysis.GlobalScope.Variables[name]?.Value; + public virtual IMember GetMember(string name) => GlobalScope.Variables[name]?.Value; public virtual IEnumerable GetMemberNames() { // drop imported modules and typing. - return Analysis.GlobalScope.Variables + return GlobalScope.Variables .Where(v => { // Instances are always fine. if (v.Value is IPythonInstance) { @@ -209,6 +210,8 @@ protected virtual void Dispose(bool disposing) { _disposeToken.TryMarkDisposed(); var analyzer = Services.GetService(); analyzer.RemoveAnalysis(this); + _parseCts?.Dispose(); + _linkedParseCts?.Dispose(); } #endregion diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 244443906..fe51b0f87 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -36,6 +36,7 @@ namespace Microsoft.Python.Analysis.Modules.Resolution { internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement { private readonly ConcurrentDictionary _specialized = new ConcurrentDictionary(); + private IModuleDatabaseService _dbService; private IRunningDocumentTable _rdt; public MainModuleResolution(string root, IServiceContainer services) @@ -76,11 +77,6 @@ protected override IPythonModule CreateModule(string name) { } } - var dbs = _services.GetService(); - if (dbs != null && dbs.TryCreateModule(name, moduleImport.ModulePath, out var m) != ModuleStorageState.DoesNotExist && m != null) { - return m; - } - // If there is a stub, make sure it is loaded and attached // First check stub next to the module. if (!TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) { @@ -156,8 +152,17 @@ public IPythonModule SpecializeModule(string name, Func s /// /// Returns specialized module, if any. /// - public IPythonModule GetSpecializedModule(string name) - => _specialized.TryGetValue(name, out var module) ? module : null; + public IPythonModule GetSpecializedModule(string fullName, string modulePath = null) { + if (_specialized.TryGetValue(fullName, out var module)) { + return module; + } + var dbs = GetDbService(); + if (dbs != null && dbs.TryCreateModule(fullName, modulePath, out module) != ModuleStorageState.DoesNotExist && module != null) { + SpecializeModule(fullName, s => module); + return module; + } + return null; + } internal async Task LoadBuiltinTypesAsync(CancellationToken cancellationToken = default) { var analyzer = _services.GetService(); @@ -237,5 +242,8 @@ private bool TryCreateModuleStub(string name, string modulePath, out IPythonModu private IRunningDocumentTable GetRdt() => _rdt ?? (_rdt = _services.GetService()); + + private IModuleDatabaseService GetDbService() + => _dbService ?? (_dbService = _services.GetService()); } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index 06dfe6802..b532f4f5e 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -21,7 +21,6 @@ using Microsoft.Python.Analysis.Caching; using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Core.Interpreter; -using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; diff --git a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs index e92e34b48..fdf4b7a66 100644 --- a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs @@ -51,9 +51,9 @@ public string Description { } public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) - => _getter?.Call(args, instance?.GetPythonType() ?? DeclaringType); + => _getter.Call(args, instance?.GetPythonType() ?? DeclaringType); - public IMember ReturnType => _getter?.Call(ArgumentSet.WithoutContext, DeclaringType)?.GetPythonType(); + public IMember ReturnType => _getter?.Call(ArgumentSet.WithoutContext, DeclaringType); #endregion internal void AddOverload(IPythonFunctionOverload overload) => _getter = _getter ?? overload; diff --git a/src/Caching/Test/Files/Requests.json b/src/Caching/Test/Files/Requests.json index 84f951687..fb7bb1831 100644 --- a/src/Caching/Test/Files/Requests.json +++ b/src/Caching/Test/Files/Requests.json @@ -208,7 +208,7 @@ "Name": "Session" }, { - "Value": "i:typing:Any", + "Value": "i:requests.structures:LookupDict", "Id": 753305199, "Name": "codes" },