diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs index cd4c7611b..fde0ffe46 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs @@ -63,57 +63,14 @@ private IMember GetValueFromGeneric(IMember target, Expression expr) { /// /// Returns whether the arguments to Generic are valid /// - private bool GenericClassParameterValid(IReadOnlyList genericTypeArgs, IReadOnlyList args, Expression expr) { - // All arguments to Generic must be type parameters - // e.g. Generic[T, str] throws a runtime error - if (genericTypeArgs.Count != args.Count) { - ReportDiagnostics(Module.Uri, new DiagnosticsEntry( - Resources.GenericNotAllTypeParameters, - GetLocation(expr).Span, - ErrorCodes.TypingGenericArguments, - Severity.Warning, - DiagnosticSource.Analysis)); - return false; - } - - // All arguments to Generic must be distinct - if (genericTypeArgs.Distinct().Count() != genericTypeArgs.Count) { - ReportDiagnostics(Module.Uri, new DiagnosticsEntry( - Resources.GenericNotAllUnique, - GetLocation(expr).Span, - ErrorCodes.TypingGenericArguments, - Severity.Warning, - DiagnosticSource.Analysis)); - return false; - } - - return true; - } /// /// Given generic type and list of arguments in the expression like /// Mapping[T1, int, ...] or Mapping[str, int] where Mapping inherits from Generic[K,T] creates generic class base /// (if the former) on specific type (if the latter). /// - private IMember CreateSpecificTypeFromIndex(IGenericType gt, IReadOnlyList args, Expression expr) { - var genericTypeArgs = args.OfType().ToArray(); - - if (gt.Name.EqualsOrdinal("Generic")) { - if (!GenericClassParameterValid(genericTypeArgs, args, expr)) { - return UnknownType; - } - - // Generic[T1, T2, ...] expression. Create generic base for the class. - return new GenericClassParameter(genericTypeArgs, Module); - } - - // For other types just use supplied arguments - if (args.Count > 0) { - return gt.CreateSpecificType(new ArgumentSet(args, expr, this)); - } - - return UnknownType; - } + private IMember CreateSpecificTypeFromIndex(IGenericType gt, IReadOnlyList args, Expression expr) + => args.Count > 0 ? gt.CreateSpecificType(new ArgumentSet(args, expr, this)) : UnknownType; private IReadOnlyList EvaluateIndex(IndexExpression expr) { var indices = new List(); diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index e8aba3904..8aab07e9c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -16,7 +16,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using System.Xml.Serialization; using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; @@ -246,118 +245,133 @@ private void MergeStub() { var sourceType = sourceVar?.Value.GetPythonType(); // If stub says 'Any' but we have better type, keep the current type. - if (!IsStubBetterType(sourceType, stubType)) { + if (stubType.DeclaringModule is TypingModule && stubType.Name == "Any") { continue; } + TryReplaceMember(v, sourceType, stubType); + } + + UpdateVariables(); + } + + private void TryReplaceMember(IVariable v, IPythonType sourceType, IPythonType stubType) { + // If type does not exist in module, but exists in stub, declare it unless it is an import. + // If types are the classes, take class from the stub, then add missing members. + // Otherwise, replace type from one from the stub. + switch (sourceType) { + case null: + // Nothing in sources, but there is type in the stub. Declare it. + if (v.Source == VariableSource.Declaration || v.Source == VariableSource.Generic) { + Eval.DeclareVariable(v.Name, v.Value, v.Source); + } + break; - // If type does not exist in module, but exists in stub, declare it unless it is an import. - // If types are the classes, merge members. Otherwise, replace type from one from the stub. - switch (sourceType) { - case null: - // Nothing in sources, but there is type in the stub. Declare it. - if (v.Source == VariableSource.Declaration || v.Source == VariableSource.Generic) { - Eval.DeclareVariable(v.Name, v.Value, v.Source); + case PythonClassType sourceClass when Module.Equals(sourceClass.DeclaringModule): + // Transfer documentation first so we get class documentation + // that came from class definition win over one that may + // come from __init__ during the member merge below. + TransferDocumentationAndLocation(sourceClass, stubType); + + // Replace the class entirely since stub members may use generic types + // and the class definition is important. We transfer missing members + // from the original class to the stub. + Eval.DeclareVariable(v.Name, v.Value, v.Source); + + // Go through source class members and pick those that are + // not present in the stub class. + foreach (var name in sourceClass.GetMemberNames()) { + + var sourceMember = sourceClass.GetMember(name); + if (sourceMember.IsUnknown()) { + continue; // Anything is better than unknowns. } - break; + var sourceMemberType = sourceMember?.GetPythonType(); - case PythonClassType cls when Module.Equals(cls.DeclaringModule): - // Transfer documentation first so we get class documentation - // that came from class definition win over one that may - // come from __init__ during the member merge below. - TransferDocumentationAndLocation(sourceType, stubType); + var stubMember = stubType.GetMember(name); + var stubMemberType = stubMember.GetPythonType(); - // If class exists and belongs to this module, add or replace - // its members with ones from the stub, preserving documentation. // Don't augment types that do not come from this module. - // Do not replace __class__ since it has to match class type and we are not - // replacing class type, we are only merging members. - foreach (var name in stubType.GetMemberNames().Except(new[] { "__class__", "__base__", "__bases__", "__mro__", "mro" })) { - var stubMember = stubType.GetMember(name); - var member = cls.GetMember(name); - - var memberType = member?.GetPythonType(); - var stubMemberType = stubMember.GetPythonType(); - - if (sourceType.IsBuiltin || stubType.IsBuiltin) { - // If stub type does not have an immediate member such as __init__() and - // rather have it inherited from object, we do not want to use the inherited - // since current class may either have its own of inherits it from the object. - continue; - } - - if (stubMemberType?.MemberType == PythonMemberType.Method && stubMemberType?.DeclaringModule.ModuleType == ModuleType.Builtins) { - // Leave methods coming from object at the object and don't copy them into the derived class. - } - - if (!IsStubBetterType(memberType, stubMemberType)) { - continue; - } - - // Get documentation from the current type, if any, since stubs - // typically do not contain documentation while scraped code does. - TransferDocumentationAndLocation(memberType, stubMemberType); - cls.AddMember(name, stubMember, overwrite: true); + if (sourceType.IsBuiltin || stubType.IsBuiltin) { + // If source type does not have an immediate member such as __init__() and + // rather have it inherited from object, we do not want to use the inherited + // since stub class may either have its own of inherits it from the object. + continue; } - break; - case IPythonModule _: - // We do not re-declare modules. - break; - - default: - var stubModule = stubType.DeclaringModule; - if (stubType is IPythonModule || stubModule.ModuleType == ModuleType.Builtins) { - // Modules members that are modules should remain as they are, i.e. os.path - // should remain library with its own stub attached. - break; + if (stubMemberType?.MemberType == PythonMemberType.Method && stubMemberType?.DeclaringModule.ModuleType == ModuleType.Builtins) { + // Leave methods coming from object at the object and don't copy them into the derived class. } - // We do not re-declaring variables that are imported. - if (v.Source == VariableSource.Declaration) { - // Re-declare variable with the data from the stub. - TransferDocumentationAndLocation(sourceType, stubType); - // TODO: choose best type between the scrape and the stub. Stub probably should always win. - var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? v.Source; - Eval.DeclareVariable(v.Name, v.Value, source); + + // Get documentation from the current type, if any, since stubs + // typically do not contain documentation while scraped code does. + TransferDocumentationAndLocation(sourceMemberType, stubMemberType); + + // If stub says 'Any' but we have better type, use member from the original class. + if (stubMember != null && !(stubType.DeclaringModule is TypingModule && stubType.Name == "Any")) { + continue; // Stub already have the member, don't replace. } + (stubType as PythonType)?.AddMember(name, stubMember, overwrite: true); + } + break; + + case IPythonModule _: + // We do not re-declare modules. + break; + + default: + var stubModule = stubType.DeclaringModule; + if (stubType is IPythonModule || stubModule.ModuleType == ModuleType.Builtins) { + // Modules members that are modules should remain as they are, i.e. os.path + // should remain library with its own stub attached. break; - } + } + // We do not re-declaring variables that are imported. + if (v.Source == VariableSource.Declaration) { + // Re-declare variable with the data from the stub. + TransferDocumentationAndLocation(sourceType, stubType); + // TODO: choose best type between the scrape and the stub. Stub probably should always win. + var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? v.Source; + Eval.DeclareVariable(v.Name, v.Value, source); + } + + break; } } - private static bool IsStubBetterType(IPythonType sourceType, IPythonType stubType) { - if (stubType.IsUnknown()) { - // Do not use worse types than what is in the module. - return false; - } - if (sourceType.IsUnknown()) { - return true; // Anything is better than unknowns. - } - if (sourceType.DeclaringModule.ModuleType == ModuleType.Specialized) { - // Types in specialized modules should never be touched. - return false; - } - if (stubType.DeclaringModule is TypingModule && stubType.Name == "Any") { - // If stub says 'Any' but we have better type, keep the current type. - return false; - } - // Types should be compatible except it is allowed to replace function by a class. - // Consider stub of threading that replaces def RLock(): by class RLock(). - // Similarly, in _json, make_scanner function is replaced by a class. - if (sourceType.MemberType == PythonMemberType.Function && stubType.MemberType == PythonMemberType.Class) { - return true; - } - // Random replaces method (variable) by a function. - if (sourceType.MemberType == PythonMemberType.Method && stubType.MemberType == PythonMemberType.Function) { - return true; + private void UpdateVariables() { + // Second pass: if we replaced any classes by new from the stub, we need to update + // variables that may still be holding old content. For example, ctypes + // declares 'c_voidp = c_void_p' so when we replace 'class c_void_p' + // by class from the stub, we need to go and update 'c_voidp' variable. + foreach (var v in GlobalScope.Variables) { + var variableType = v.Value.GetPythonType(); + if (!variableType.DeclaringModule.Equals(Module) && !variableType.DeclaringModule.Equals(Module.Stub)) { + continue; + } + // Check if type that the variable references actually declared here. + var typeInScope = GlobalScope.Variables[variableType.Name].GetPythonType(); + if (typeInScope == null || variableType == typeInScope) { + continue; + } + + if (v.Value == variableType) { + Eval.DeclareVariable(v.Name, typeInScope, v.Source); + } else if (v.Value is IPythonInstance) { + Eval.DeclareVariable(v.Name, new PythonInstance(typeInScope), v.Source); + } } - return sourceType.MemberType == stubType.MemberType; } private static void TransferDocumentationAndLocation(IPythonType s, IPythonType d) { - if (s.IsUnknown() || d.IsBuiltin || s.IsBuiltin) { + if (s.IsUnknown() || s.IsBuiltin || d == null || d.IsBuiltin) { return; // Do not transfer location of unknowns or builtins } + + if (d.DeclaringModule != s.DeclaringModule.Stub) { + return; // Do not change unrelated types. + } + // Documentation and location are always get transferred from module type // to the stub type and never the other way around. This makes sure that // we show documentation from the original module and goto definition @@ -368,7 +382,7 @@ private static void TransferDocumentationAndLocation(IPythonType s, IPythonType var transferDoc = true; if (src is PythonClassType srcClass && dst is PythonClassType dstClass) { // Higher lever source wins - if(srcClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Class || + if (srcClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Class || (srcClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Init && dstClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Base)) { dstClass.SetDocumentation(srcClass.Documentation); transferDoc = false; diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs index 3f26a08a7..1901366ff 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs @@ -54,7 +54,6 @@ public void EvaluateClass() { var bases = ProcessBases(); _class.SetBases(bases); - _class.DecideGeneric(); // Declare __class__ variable in the scope. Eval.DeclareVariable("__class__", _class, VariableSource.Declaration); ProcessClassBody(); diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 365a0bd12..f0ea79c9f 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -73,7 +73,7 @@ public override void Evaluate() { v => v.GetPythonType() == null && v.GetPythonType() == null) ) { - ((VariableCollection)Eval.CurrentScope.Variables).Clear(); + ((VariableCollection)Eval.CurrentScope.Variables).Clear(); } } } diff --git a/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs b/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs index d74f9bf61..6699f54d8 100644 --- a/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs @@ -13,9 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. - using System.Collections.Generic; -using System.ComponentModel; using System.Linq; using System.Security.Cryptography.X509Certificates; using Microsoft.Python.Analysis.Analyzer.Evaluation; diff --git a/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs index 4c6ec4e7f..36f3ac42f 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs @@ -39,9 +39,9 @@ public static IScope GetScope(this IPythonFunctionType f) { return gs?.TraverseBreadthFirst(s => s.Children).FirstOrDefault(s => s.Node == f.FunctionDefinition); } - public static string GetQualifiedName(this IPythonClassMember cm) { + public static string GetQualifiedName(this IPythonClassMember cm, string baseName = null) { var s = new Stack(); - s.Push(cm.Name); + s.Push(baseName ?? cm.Name); for (var p = cm.DeclaringType as IPythonClassMember; p != null; p = p.DeclaringType as IPythonClassMember) { s.Push(p.Name); } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index 4c2c557d1..e8e1ca8c4 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -17,7 +17,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Threading; using Microsoft.Python.Analysis.Caching; using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Core.Interpreter; @@ -35,7 +34,6 @@ internal abstract class ModuleResolutionBase { protected readonly IFileSystem _fs; protected readonly ILogger _log; protected readonly IUIService _ui; - protected readonly bool _requireInitPy; protected ConcurrentDictionary Modules { get; } = new ConcurrentDictionary(); protected PathResolver PathResolver { get; set; } @@ -50,9 +48,7 @@ protected ModuleResolutionBase(string root, IServiceContainer services) { _fs = services.GetService(); _log = services.GetService(); _ui = services.GetService(); - - _requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_interpreter.Configuration.Version); - } + } public string Root { get; protected set; } public ImmutableArray InterpreterPaths { get; protected set; } = ImmutableArray.Empty; diff --git a/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs b/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs index 7815fe4a8..7efa0a679 100644 --- a/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs +++ b/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs @@ -20,21 +20,22 @@ using Microsoft.Python.Analysis.Types.Collections; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Analysis.Values.Collections; +using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Specializations { public static class BuiltinsSpecializations { - public static IMember Identity(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember Identity(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var args = argSet.Values(); return args.Count > 0 ? args.FirstOrDefault(a => !a.IsUnknown()) ?? args[0] : null; } - public static IMember TypeInfo(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember TypeInfo(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var args = argSet.Values(); var t = args.Count > 0 ? args[0].GetPythonType() : module.Interpreter.GetBuiltinType(BuiltinTypeId.Type); return t.ToBound(); } - public static IMember Iterator(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember Iterator(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var args = argSet.Values(); if (args.Count > 0) { if (args[0] is IPythonCollection seq) { @@ -48,21 +49,21 @@ public static IMember Iterator(IPythonModule module, IPythonFunctionOverload ove return null; } - public static IMember List(IPythonInterpreter interpreter, IPythonFunctionOverload overload, IArgumentSet argSet) + public static IMember List(IPythonInterpreter interpreter, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) => PythonCollectionType.CreateList(interpreter.ModuleResolution.BuiltinsModule, argSet); - public static IMember ListOfStrings(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember ListOfStrings(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var type = new TypingListType("List", module.Interpreter.GetBuiltinType(BuiltinTypeId.Str), module.Interpreter, false); return new TypingList(type); } - public static IMember DictStringToObject(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember DictStringToObject(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var str = module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); var obj = module.Interpreter.GetBuiltinType(BuiltinTypeId.Object); var type = new TypingDictionaryType("Dict", str, obj, module.Interpreter, false); return new TypingDictionary(type); } - public static IMember Next(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember Next(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var args = argSet.Values(); return args.Count > 0 && args[0] is IPythonIterator it ? it.Next : null; } @@ -76,7 +77,7 @@ public static IMember __iter__(IPythonInterpreter interpreter, BuiltinTypeId con return fn; } - public static IMember Range(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember Range(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var args = argSet.Values(); if (args.Count > 0) { var type = new PythonCollectionType(BuiltinTypeId.List, module.Interpreter.ModuleResolution.BuiltinsModule, false); @@ -85,12 +86,12 @@ public static IMember Range(IPythonModule module, IPythonFunctionOverload overlo return null; } - public static IMember CollectionItem(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember CollectionItem(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var args = argSet.Values(); return args.Count > 0 && args[0] is PythonCollection c ? c.Contents.FirstOrDefault() : null; } - public static IMember Open(IPythonModule declaringModule, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember Open(IPythonModule declaringModule, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var mode = argSet.GetArgumentValue("mode"); var binary = false; @@ -118,7 +119,7 @@ public static IMember Open(IPythonModule declaringModule, IPythonFunctionOverloa return returnType != null ? new PythonInstance(returnType) : null; } - public static IMember GetAttr(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember GetAttr(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { // TODO: Try __getattr__ first; this may not be as reliable in practice // given we could be assuming that __getattr__ always returns the same type, // which is incorrect more often than not. diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassBase.cs similarity index 95% rename from src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassParameter.cs rename to src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassBase.cs index 10cbc03f9..e5ca22b09 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassParameter.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassBase.cs @@ -20,7 +20,7 @@ namespace Microsoft.Python.Analysis.Specializations.Typing { /// /// Represents Generic[T1, T2, ...]. Used as a base class to generic classes. /// - public interface IGenericClassParameter: IPythonType { + public interface IGenericClassBase: IPythonType { /// /// List of T1, T2, ... generic type parameters /// diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericType.cs index 79e6fbc27..195194d09 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericType.cs @@ -26,7 +26,7 @@ public interface IGenericType : IPythonTemplateType { /// Type parameters such as in Tuple[T1, T2. ...] or /// Generic[_T1, _T2, ...] as returned by TypeVar. /// - IReadOnlyList Parameters { get; } + IReadOnlyList FormalGenericParameters { get; } bool IsGeneric { get; } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs index 17d5acbe8..bc22e2a3a 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs @@ -20,7 +20,6 @@ namespace Microsoft.Python.Analysis.Specializations.Typing { /// Represents typing.NamedTuple. /// public interface ITypingNamedTupleType : ITypingTupleType { - string TupleName { get; } IReadOnlyList ItemNames { get; } } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs new file mode 100644 index 000000000..c16728806 --- /dev/null +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs @@ -0,0 +1,87 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Diagnostics; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing; +using ErrorCodes = Microsoft.Python.Analysis.Diagnostics.ErrorCodes; + +namespace Microsoft.Python.Analysis.Specializations.Typing.Types { + /// + /// Represents Generic[T1, T2, ...] base. When class is instantiated + /// or methods evaluated, class generic parameters are matched to + /// generic type parameters from TypeVar. + /// + internal sealed class GenericClassBase : PythonClassType, IGenericClassBase { + internal GenericClassBase(IReadOnlyList typeArgs, IPythonInterpreter interpreter) + : base("Generic", new Location(interpreter.ModuleResolution.GetSpecializedModule("typing"))) { + TypeParameters = typeArgs ?? Array.Empty(); + } + + #region IPythonType + public override PythonMemberType MemberType => PythonMemberType.Generic; + public override string Documentation => Name; + #endregion + + #region IPythonClassType + public override bool IsGeneric => true; + public override IReadOnlyDictionary GenericParameters + => TypeParameters.ToDictionary(tp => tp, tp => tp as IPythonType ?? UnknownType); + + public override IPythonType CreateSpecificType(IArgumentSet args) { + if (!GenericClassParameterValid(args)) { + return UnknownType; + } + return new GenericClassBase(args.Arguments.Select(a => a.Value).OfType().ToArray(), DeclaringModule.Interpreter); + } + + #endregion + + public IReadOnlyList TypeParameters { get; } + + private bool GenericClassParameterValid(IArgumentSet args) { + var genericTypeArgs = args.Values().ToArray(); + var allArgs = args.Values().ToArray(); + // All arguments to Generic must be type parameters + // e.g. Generic[T, str] throws a runtime error + var e = args.Eval; + if (genericTypeArgs.Length != allArgs.Length) { + e.ReportDiagnostics(args.Eval.Module.Uri, new DiagnosticsEntry( + Resources.GenericNotAllTypeParameters, + e.GetLocation(args.Expression).Span, + ErrorCodes.TypingGenericArguments, + Severity.Warning, + DiagnosticSource.Analysis)); + return false; + } + + // All arguments to Generic must be distinct + if (genericTypeArgs.Distinct().ToArray().Length != genericTypeArgs.Length) { + e.ReportDiagnostics(args.Eval.Module.Uri, new DiagnosticsEntry( + Resources.GenericNotAllUnique, + e.GetLocation(args.Expression).Span, + ErrorCodes.TypingGenericArguments, + Severity.Warning, + DiagnosticSource.Analysis)); + return false; + } + + return true; + } + } +} diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassParameter.cs deleted file mode 100644 index 164d021d8..000000000 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassParameter.cs +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; -using Microsoft.Python.Core.Disposables; - -namespace Microsoft.Python.Analysis.Specializations.Typing.Types { - /// - /// Represents Generic[T1, T2, ...] parameter. When class is instantiated - /// or methods evaluated, class generic parameters are matched to - /// generic type parameters from TypeVar. - /// - internal sealed class GenericClassParameter : PythonClassType, IGenericClassParameter { - internal GenericClassParameter(IReadOnlyList typeArgs, IPythonModule declaringModule) - : base("Generic", new Location(declaringModule)) { - TypeParameters = typeArgs ?? new List(); - } - - public override bool IsGeneric => true; - - public override IReadOnlyDictionary GenericParameters - => TypeParameters.ToDictionary(tp => tp, tp => tp as IPythonType ?? UnknownType) ?? EmptyDictionary.Instance; - - public IReadOnlyList TypeParameters { get; } - - public override PythonMemberType MemberType => PythonMemberType.Generic; - } -} diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs index 6aa95d21f..75752d2b1 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System; -using System.Collections; using System.Collections.Generic; using System.Linq; using Microsoft.Python.Analysis.Types; @@ -26,21 +25,23 @@ namespace Microsoft.Python.Analysis.Specializations.Typing.Types { /// /// Base class for generic types and type declarations. /// - internal class SpecializedGenericType : LocatedMember, IGenericType { + internal sealed class SpecializedGenericType : LocatedMember, IGenericType { internal SpecificTypeConstructor SpecificTypeConstructor { get; } /// /// Constructs generic type with generic type parameters. Typically used /// in generic classes such as when handling Generic[_T] base. /// - public SpecializedGenericType(string name, IReadOnlyList parameters, IPythonModule declaringModule) - : this(name, declaringModule) { - Parameters = parameters ?? throw new ArgumentNullException(nameof(parameters)); + public SpecializedGenericType(string name, string qualifiedName, IReadOnlyList parameters, IPythonModule declaringModule) + : this(name, qualifiedName, declaringModule) { + FormalGenericParameters = parameters ?? throw new ArgumentNullException(nameof(parameters)); } /// /// Constructs generic type with dynamic type constructor. /// Typically used in type specialization scenarios. + /// Type created with this constructor cannot be persisted + /// since it does not have qualified name. /// /// Type name including parameters, such as Iterator[T] /// Constructor of specific types. @@ -48,20 +49,53 @@ public SpecializedGenericType(string name, IReadOnlyList /// Type id. Used in type comparisons such as when matching /// function arguments. For example, Iterator[T] normally has type id of ListIterator. /// Optional type parameters as declared by TypeVar. + /// Optional documentation. Defaults to . public SpecializedGenericType( string name, SpecificTypeConstructor specificTypeConstructor, IPythonModule declaringModule, BuiltinTypeId typeId = BuiltinTypeId.Unknown, - IReadOnlyList parameters = null - ) : this(name, declaringModule) { + IReadOnlyList parameters = null, + string documentation = null + ) : this(name, null, declaringModule) { SpecificTypeConstructor = specificTypeConstructor ?? throw new ArgumentNullException(nameof(specificTypeConstructor)); TypeId = typeId; - Parameters = parameters ?? Array.Empty(); + FormalGenericParameters = parameters ?? Array.Empty(); + Documentation = documentation ?? name; } - private SpecializedGenericType(string name, IPythonModule declaringModule) : base(declaringModule) { + /// + /// Constructs generic type with dynamic type constructor. + /// Typically used in type specialization scenarios. + /// + /// Type name including parameters, such as Iterator[T] + /// Qualified type name including parameters, such as typing:Iterator[module:T] + /// Constructor of specific types. + /// Declaring module. + /// Type id. Used in type comparisons such as when matching + /// function arguments. For example, Iterator[T] normally has type id of ListIterator. + /// Optional type parameters as declared by TypeVar. + /// Optional documentation. Defaults to . + public SpecializedGenericType( + string name, + string qualifiedName, + SpecificTypeConstructor specificTypeConstructor, + IPythonModule declaringModule, + BuiltinTypeId typeId = BuiltinTypeId.Unknown, + IReadOnlyList parameters = null, + string documentation = null + ) : this(name, qualifiedName, declaringModule) { + SpecificTypeConstructor = specificTypeConstructor ?? throw new ArgumentNullException(nameof(specificTypeConstructor)); + TypeId = typeId; + FormalGenericParameters = parameters ?? Array.Empty(); + Documentation = documentation ?? name; + } + + private SpecializedGenericType(string name, string qualifiedName, IPythonModule declaringModule) + : base(declaringModule) { Name = name ?? throw new ArgumentNullException(nameof(name)); + QualifiedName = qualifiedName ?? $"{declaringModule.Name}:{name}"; + Documentation = Name; } public override PythonMemberType MemberType => PythonMemberType.Generic; @@ -70,15 +104,15 @@ private SpecializedGenericType(string name, IPythonModule declaringModule) : bas /// Type parameters such as in Tuple[T1, T2. ...] or /// Generic[_T1, _T2, ...] as returned by TypeVar. /// - public IReadOnlyList Parameters { get; } + public IReadOnlyList FormalGenericParameters { get; } #region IPythonType public string Name { get; } - public string QualifiedName => this.GetQualifiedName(); + public string QualifiedName { get; } public IMember GetMember(string name) => null; public IEnumerable GetMemberNames() => Enumerable.Empty(); public BuiltinTypeId TypeId { get; } = BuiltinTypeId.Unknown; - public virtual string Documentation => Name; + public string Documentation { get; } public bool IsBuiltin => false; public bool IsAbstract => true; public bool IsSpecialized => true; @@ -96,8 +130,8 @@ public IMember CreateInstance(string typeName, IArgumentSet args) { : specific.CreateInstance(typeName); } - public virtual IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; - public virtual IMember Index(IPythonInstance instance, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; + public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; + public IMember Index(IPythonInstance instance, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; /// /// Creates instance of a type information with the specific diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs index 636fc7f02..c02bac873 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs @@ -27,7 +27,6 @@ namespace Microsoft.Python.Analysis.Specializations.Typing.Types { internal sealed class GenericTypeParameter : PythonType, IGenericTypeParameter { public GenericTypeParameter( string name, - IPythonModule declaringModule, IReadOnlyList constraints, object bound, object covariant, @@ -118,7 +117,7 @@ public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declari var covariant = argSet.GetArgumentValue("covariant")?.Value; var contravariant = argSet.GetArgumentValue("contravariant")?.Value; - return new GenericTypeParameter(name, declaringModule, constraints, bound, covariant, contravariant, new Location(declaringModule, location)); + return new GenericTypeParameter(name, constraints, bound, covariant, contravariant, new Location(declaringModule, location)); } private static string GetDocumentation(string name, IReadOnlyList constraints, object bound, object covariant, object contravariant) { diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs index cfad82980..2ecd670e5 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs @@ -21,32 +21,50 @@ using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Specializations.Typing.Types { internal sealed class NamedTupleType : TypingTupleType, ITypingNamedTupleType { + // Since named tuple operates as a new, separate type, we need to track + // its location rather than delegating down to the general wrapper over + // Python built-in tuple. + private sealed class NamedTupleLocatedMember: LocatedMember { + public NamedTupleLocatedMember(Location location) : base(location) { } + public override PythonMemberType MemberType => PythonMemberType.Class; + } + private readonly NamedTupleLocatedMember _locatedMember; + /// /// Creates type info for a strongly-typed tuple, such as Tuple[T1, T2, ...]. /// - public NamedTupleType(string tupleName, IReadOnlyList itemNames, IReadOnlyList itemTypes, IPythonModule declaringModule, IPythonInterpreter interpreter) - : base(itemTypes, declaringModule, interpreter) { - TupleName = tupleName ?? throw new ArgumentNullException(nameof(tupleName)); + public NamedTupleType(string tupleName, IReadOnlyList itemNames, IReadOnlyList itemTypes, IPythonModule declaringModule, IndexSpan indexSpan) + : base(itemTypes, declaringModule, declaringModule.Interpreter) { + Name = tupleName ?? throw new ArgumentNullException(nameof(tupleName)); ItemNames = itemNames; var typeNames = itemTypes.Select(t => t.IsUnknown() ? string.Empty : t.Name); var pairs = itemNames.Zip(typeNames, (name, typeName) => string.IsNullOrEmpty(typeName) ? name : $"{name}: {typeName}"); - Name = CodeFormatter.FormatSequence(tupleName, '(', pairs); + Documentation = CodeFormatter.FormatSequence(tupleName, '(', pairs); - typeNames = itemTypes.Select(t => t.IsUnknown() ? string.Empty : t.QualifiedName); - pairs = itemNames.Zip(typeNames, (name, typeName) => string.IsNullOrEmpty(typeName) ? name : $"{name}: {typeName}"); - QualifiedName = CodeFormatter.FormatSequence($"{declaringModule.Name}:{tupleName}", '(', pairs); + _locatedMember = new NamedTupleLocatedMember(new Location(declaringModule, indexSpan)); } - public string TupleName { get; } public IReadOnlyList ItemNames { get; } + #region IPythonType public override string Name { get; } - public override string QualifiedName { get; } + public override string QualifiedName => $"{DeclaringModule.Name}:{Name}"; // Named tuple name is a type name as class. public override bool IsSpecialized => true; + public override string Documentation { get; } + #endregion + + #region ILocatedMember + public override Location Location => _locatedMember.Location; + public override LocationInfo Definition => _locatedMember.Definition; + public override IReadOnlyList References => _locatedMember.References; + public override void AddReference(Location location) => _locatedMember.AddReference(location); + public override void RemoveReferences(IPythonModule module) => _locatedMember.RemoveReferences(module); + #endregion public override IMember CreateInstance(string typeName, IArgumentSet args) => new TypingTuple(this); diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingTupleType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingTupleType.cs index f393516d2..52358c16c 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingTupleType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingTupleType.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Specializations.Typing.Values; using Microsoft.Python.Analysis.Types; diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs index 5538e7aaf..c854b16b0 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs @@ -23,6 +23,7 @@ using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; +using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; @@ -61,8 +62,8 @@ private void SpecializeMembers() { // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. - o.SetReturnValueProvider((declaringModule, overload, args) - => GenericTypeParameter.FromTypeVar(args, declaringModule)); + o.SetReturnValueProvider((declaringModule, overload, args, indexSpan) + => GenericTypeParameter.FromTypeVar(args, declaringModule, indexSpan)); fn.AddOverload(o); _members["TypeVar"] = fn; @@ -75,7 +76,7 @@ private void SpecializeMembers() { o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. - o.SetReturnValueProvider((declaringModule, overload, args) => { + o.SetReturnValueProvider((declaringModule, overload, args, indexSpan) => { var a = args.Values(); return a.Count == 1 ? a[0] : Interpreter.UnknownType; }); @@ -125,15 +126,17 @@ private void SpecializeMembers() { _members["Counter"] = Specialized.Function("Counter", this, GetMemberDocumentation("Counter"), new PythonInstance(Interpreter.GetBuiltinType(BuiltinTypeId.Int))); - _members["SupportsInt"] = Interpreter.GetBuiltinType(BuiltinTypeId.Int); - _members["SupportsFloat"] = Interpreter.GetBuiltinType(BuiltinTypeId.Float); - _members["SupportsComplex"] = Interpreter.GetBuiltinType(BuiltinTypeId.Complex); - _members["SupportsBytes"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); + // TODO: make these classes that support __float__, etc per spec. + //_members["SupportsInt"] = Interpreter.GetBuiltinType(BuiltinTypeId.Int); + //_members["SupportsFloat"] = Interpreter.GetBuiltinType(BuiltinTypeId.Float); + //_members["SupportsComplex"] = Interpreter.GetBuiltinType(BuiltinTypeId.Complex); + //_members["SupportsBytes"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); _members["ByteString"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); fn = PythonFunctionType.Specialize("NamedTuple", this, GetMemberDocumentation("NamedTuple")); o = new PythonFunctionOverload(fn.Name, location); - o.SetReturnValueProvider((declaringModule, overload, args) => CreateNamedTuple(args.Values(), declaringModule)); + o.SetReturnValueProvider((declaringModule, overload, args, indexSpan) + => CreateNamedTuple(args.Values(), declaringModule, indexSpan)); fn.AddOverload(o); _members["NamedTuple"] = fn; @@ -154,7 +157,7 @@ private IPythonType SpecializeNewType(Location location) { var o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. - o.SetReturnValueProvider((interpreter, overload, args) => CreateTypeAlias(args)); + o.SetReturnValueProvider((interpreter, overload, args, indexSpan) => CreateTypeAlias(args)); o.SetParameters(new[] { new ParameterInfo("name", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.Normal, null), new ParameterInfo("tp", Interpreter.GetBuiltinType(BuiltinTypeId.Type), ParameterKind.Normal, null), @@ -279,7 +282,7 @@ private IPythonType CreateUnion(IReadOnlyList typeArgs) { return Interpreter.UnknownType; } - private IPythonType CreateNamedTuple(IReadOnlyList typeArgs, IPythonModule declaringModule) { + private IPythonType CreateNamedTuple(IReadOnlyList typeArgs, IPythonModule declaringModule, IndexSpan indexSpan) { if (typeArgs.Count != 2) { // TODO: report wrong number of arguments return Interpreter.UnknownType; @@ -324,7 +327,7 @@ private IPythonType CreateNamedTuple(IReadOnlyList typeArgs, IPythonMod itemNames.Add(itemName2); itemTypes.Add(c.Contents[1].GetPythonType()); } - return TypingTypeFactory.CreateNamedTupleType(Interpreter, tupleName, itemNames, itemTypes, declaringModule); + return TypingTypeFactory.CreateNamedTupleType(tupleName, itemNames, itemTypes, declaringModule, indexSpan); } private IPythonType CreateOptional(IReadOnlyList typeArgs) { @@ -363,7 +366,7 @@ private IPythonType CreateGenericClassParameter(IReadOnlyList typeA if (typeArgs.Count > 0) { var typeDefs = typeArgs.OfType().ToArray(); if (typeDefs.Length == typeArgs.Count) { - return new GenericClassParameter(typeDefs, this); + return new GenericClassBase(typeDefs, Interpreter); } else { // TODO: report argument mismatch } @@ -372,10 +375,14 @@ private IPythonType CreateGenericClassParameter(IReadOnlyList typeA return Interpreter.UnknownType; } - private IPythonType ToGenericTemplate(string typeName, IReadOnlyList typeArgs, BuiltinTypeId typeId) - => _members[typeName] is SpecializedGenericType gt - ? new SpecializedGenericType(CodeFormatter.FormatSequence(typeName, '[', typeArgs), gt.SpecificTypeConstructor, this, typeId, - typeArgs.OfType().ToList()) - : Interpreter.UnknownType; + private IPythonType ToGenericTemplate(string typeName, IReadOnlyList typeArgs, BuiltinTypeId typeId) { + if (_members[typeName] is SpecializedGenericType gt) { + var name = CodeFormatter.FormatSequence(typeName, '[', typeArgs); + var qualifiedName = CodeFormatter.FormatSequence($"typing:{typeName}", '[', typeArgs.Select(t => t.QualifiedName)); + var args = typeArgs.OfType().ToList(); + return new SpecializedGenericType(name, qualifiedName, gt.SpecificTypeConstructor, this, typeId, args); + } + return Interpreter.UnknownType; + } } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingTypeFactory.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingTypeFactory.cs index 540615aac..d78037e8a 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingTypeFactory.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingTypeFactory.cs @@ -19,6 +19,7 @@ using Microsoft.Python.Analysis.Specializations.Typing.Values; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Utilities; +using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Specializations.Typing { internal static class TypingTypeFactory { @@ -55,11 +56,8 @@ public static ITypingListType CreateItemsViewType(IPythonInterpreter interpreter public static IPythonType CreateUnionType(IPythonInterpreter interpreter, IReadOnlyList types, IPythonModule declaringModule) => new PythonUnionType(types.Select(a => a.GetPythonType()), declaringModule); - public static ITypingNamedTupleType CreateNamedTupleType(IPythonInterpreter interpreter, string tupleName, IReadOnlyList itemNames, IReadOnlyList itemTypes, IPythonModule declaringModule) - => new NamedTupleType(tupleName, itemNames, itemTypes, declaringModule, interpreter); - - public static IPythonType CreateOptionalType(IPythonModule declaringModule, IPythonType type) - => new OptionalType(declaringModule, type); + public static ITypingNamedTupleType CreateNamedTupleType(string tupleName, IReadOnlyList itemNames, IReadOnlyList itemTypes, IPythonModule declaringModule, IndexSpan indexSpan) + => new NamedTupleType(tupleName, itemNames, itemTypes, declaringModule, indexSpan); public static IPythonType CreateType(IPythonModule declaringModule, IPythonType type) => new TypingType(declaringModule, type); diff --git a/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs b/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs index e8e6f598f..4ea5c59ae 100644 --- a/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs +++ b/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs @@ -16,9 +16,11 @@ using System; using System.Collections.Generic; using System.Linq; +using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Analysis.Values.Collections; using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types.Collections { /// @@ -63,8 +65,23 @@ public override IMember Call(IPythonInstance instance, string memberName, IArgum public override IMember Index(IPythonInstance instance, IArgumentSet args) => (instance as IPythonCollection)?.Index(args) ?? UnknownType; + + public IPythonType CreateSpecificType(IArgumentSet typeArguments) { + throw new NotImplementedException(); + } + #endregion + #region IPythonClassType + public IPythonType DeclaringType => (InnerType as IPythonClassType)?.DeclaringType; + public IReadOnlyList FormalGenericParameters => (InnerType as IPythonClassType)?.FormalGenericParameters ?? Array.Empty(); + public bool IsGeneric => (InnerType as IPythonClassType)?.IsGeneric == true; + public ClassDefinition ClassDefinition => (InnerType as IPythonClassType)?.ClassDefinition; + public IReadOnlyList Mro => (InnerType as IPythonClassType)?.Mro ?? Array.Empty(); + public IReadOnlyList Bases => (InnerType as IPythonClassType)?.Bases ?? Array.Empty(); + public IReadOnlyDictionary GenericParameters + => (InnerType as IPythonClassType)?.GenericParameters ?? EmptyDictionary.Instance; + #endregion public static IPythonCollection CreateList(IPythonModule declaringModule, IArgumentSet args) { var exact = true; @@ -107,7 +124,7 @@ public static IPythonCollection CreateSet(IPythonModule declaringModule, IReadOn return new PythonCollection(collectionType, contents, flatten, exact: exact); } - public override bool Equals(object obj) + public override bool Equals(object obj) => obj is IPythonType pt && (PythonTypeComparer.Instance.Equals(pt, this) || PythonTypeComparer.Instance.Equals(pt, InnerType)); public override int GetHashCode() => PythonTypeComparer.Instance.GetHashCode(this); } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonCollectionType.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonCollectionType.cs index 000fa0987..115c6168b 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonCollectionType.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonCollectionType.cs @@ -19,7 +19,7 @@ namespace Microsoft.Python.Analysis.Types { /// /// Represents instance of a collection. /// - public interface IPythonCollectionType : IPythonType { + public interface IPythonCollectionType : IPythonClassType { /// /// Type of the collection iterator. /// diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs index a4101de0a..e8716c754 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs @@ -15,7 +15,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Utilities; @@ -28,10 +27,14 @@ internal partial class PythonClassType { private readonly ReentrancyGuard _genericSpecializationGuard = new ReentrancyGuard(); private readonly ReentrancyGuard _genericResolutionGuard = new ReentrancyGuard(); - private bool _isGeneric; + private string _nameWithParameters; // Name of the class with generic parameters like abc[int]. + private string _qualifiedNameWithParameters; // Qualified name with qualified parameter names for persistence. + private Dictionary _specificTypeCache; - private Dictionary _genericParameters; - private IReadOnlyList _parameters = new List(); + // Actual assigned parameters for generic class. + private Dictionary _genericActualParameters; + // Not yet set generic parameters i.e. TypeVars from Generic[T1, T2, ...]. + private IReadOnlyList _genericParameters = new List(); #region IGenericType /// @@ -41,10 +44,9 @@ internal partial class PythonClassType { /// {T, T} /// Where T is a generic type parameter that needs to be filled in by the class /// - public virtual IReadOnlyList Parameters => _parameters ?? Array.Empty(); - - public virtual bool IsGeneric => _isGeneric; + public virtual IReadOnlyList FormalGenericParameters => _genericParameters ?? Array.Empty(); + public virtual bool IsGeneric { get; private set; } #endregion /// @@ -59,7 +61,7 @@ internal partial class PythonClassType { /// B[int] defines the type parameter T to be of type int and type parameter U to be type str. /// B[int] inherits from A[int, str] /// - public IPythonType CreateSpecificType(IArgumentSet args) { + public virtual IPythonType CreateSpecificType(IArgumentSet args) { lock (_genericParameterLock) { var genericTypeParameters = GetTypeParameters(); var newBases = new List(); @@ -69,14 +71,14 @@ public IPythonType CreateSpecificType(IArgumentSet args) { var genericTypeToSpecificType = GetSpecificTypes(args, genericTypeParameters, newBases); var classType = new PythonClassType(BaseName, new Location(DeclaringModule)); + classType.SetDocumentation(Documentation); + // Storing generic parameters allows methods returning generic types // to know what type parameter returns what specific type StoreGenericParameters(classType, genericTypeParameters, genericTypeToSpecificType); // Set generic name - if (!classType._genericParameters.IsNullOrEmpty()) { - classType._genericName = CodeFormatter.FormatSequence(BaseName, '[', classType._genericParameters.Values); - } + classType.SetNames(); // Locking so threads can only access class after it's been initialized // Store generic parameters first so name updates correctly, then check if class type has been cached @@ -86,7 +88,7 @@ public IPythonType CreateSpecificType(IArgumentSet args) { } _specificTypeCache[classType.Name] = classType; - // Prevent reentrancy when resolving generic class where method may be returning instance of type of the same class. + // Prevent re-entrancy when resolving generic class where method may be returning instance of type of the same class. // e.g // class C(Generic[T]): // def tmp(self): @@ -101,13 +103,13 @@ public IPythonType CreateSpecificType(IArgumentSet args) { // Bases can be null when not set var bases = Bases ?? Array.Empty(); // Get declared generic class parameters, i.e. Generic[T1, T2, ...], Optional[Generic[T1, ...]] - var genericClassParameters = bases.OfType().ToArray(); + var genericClassParameters = bases.OfType().ToArray(); // Get list of bases that are generic but not generic class parameters, e.g A[T], B[T] but not Generic[T1, T2] var genericTypeBases = bases.Except(genericClassParameters).OfType().Where(g => g.IsGeneric).ToArray(); - // Removing all generic bases, and will only specialize genericTypeBases, remove generic class paramters entirely - // We remove generic class paramters entirely because the type information is now stored in GenericParameters field + // Removing all generic bases, and will only specialize genericTypeBases, remove generic class parameters entirely + // We remove generic class parameters entirely because the type information is now stored in GenericParameters field // We still need generic bases so we can specialize them var specificBases = bases.Except(genericTypeBases).Except(genericClassParameters).ToList(); @@ -115,7 +117,7 @@ public IPythonType CreateSpecificType(IArgumentSet args) { foreach (var gt in genericTypeBases) { // Look through generic type bases and see if any of their required type parameters // have received a specific type, and if so create specific type - var st = gt.Parameters + var st = gt.FormalGenericParameters .Select(p => classType.GenericParameters.TryGetValue(p, out var t) ? t : null) .Where(p => !p.IsUnknown()) .ToArray(); @@ -129,9 +131,7 @@ public IPythonType CreateSpecificType(IArgumentSet args) { // Set specific class bases classType.SetBases(specificBases.Concat(newBases)); - // Now that parameters are set, check if class is generic - classType._parameters = classType._genericParameters.Values.Distinct().OfType().ToList(); - classType.DecideGeneric(); + classType.SetGenericParameters(); // Transfer members from generic to specific type. SetClassMembers(classType, args); } @@ -144,13 +144,13 @@ public IPythonType CreateSpecificType(IArgumentSet args) { /// private IGenericTypeParameter[] GetTypeParameters() { // Case when updating with specific type and already has type parameters, return them - if (!Parameters.IsNullOrEmpty()) { - return Parameters.ToArray(); + if (!FormalGenericParameters.IsNullOrEmpty()) { + return FormalGenericParameters.ToArray(); } var bases = Bases ?? Array.Empty(); var fromBases = new HashSet(); - var genericClassParameter = bases.OfType().FirstOrDefault(); + var genericClassParameter = bases.OfType().FirstOrDefault(); // If Generic[...] is present, ordering of type variables is determined from that if (genericClassParameter != null && genericClassParameter.TypeParameters != null) { @@ -158,8 +158,8 @@ private IGenericTypeParameter[] GetTypeParameters() { } else { // otherwise look at the generic class bases foreach (var gt in bases.OfType()) { - if (gt.Parameters != null) { - fromBases.UnionWith(gt.Parameters); + if (gt.FormalGenericParameters != null) { + fromBases.UnionWith(gt.FormalGenericParameters); } } } @@ -216,7 +216,7 @@ private IReadOnlyDictionary GetSpecificTypes // will have BultinTypeId.Dict and we can figure out specific types from // the content of the collection. var b = _bases.OfType().Where(g => g.IsGeneric).FirstOrDefault(x => x.TypeId == type.TypeId); - if (b != null && !b.Parameters.IsNullOrEmpty()) { + if (b != null && !b.FormalGenericParameters.IsNullOrEmpty()) { newBases.Add(type); // Optimistically assign argument types if they match. // Handle common cases directly. @@ -254,22 +254,21 @@ private IReadOnlyDictionary GetSpecificTypes } /// - /// Points the generic type parameter in class type to their corresponding specific type (or a generic - /// type parameter if no specific type was provided) + /// Points the generic type parameter in class type to their corresponding specific type + /// (or a generic type parameter if no specific type was provided) /// - private void StoreGenericParameters(PythonClassType classType, IGenericTypeParameter[] genericParameters, IReadOnlyDictionary genericToSpecificTypes) { + internal void StoreGenericParameters(PythonClassType specificClassType, IGenericTypeParameter[] genericParameters, IReadOnlyDictionary genericToSpecificTypes) { // copy original generic parameters over and try to fill them in - classType._genericParameters = new Dictionary(GenericParameters.ToDictionary(k => k.Key, k => k.Value)); + specificClassType._genericActualParameters = new Dictionary(GenericParameters.ToDictionary(k => k.Key, k => k.Value)); // Case when creating a new specific class type - if (Parameters.Count == 0) { + if (FormalGenericParameters.Count == 0) { // Assign class type generic type parameters to specific types - for (var i = 0; i < genericParameters.Length; i++) { - var gb = genericParameters[i]; - classType._genericParameters[gb] = genericToSpecificTypes.TryGetValue(gb, out var v) ? v : null; + foreach (var gb in genericParameters) { + specificClassType._genericActualParameters[gb] = genericToSpecificTypes.TryGetValue(gb, out var v) ? v : null; } } else { - // When Parameters field is not empty then need to update generic parameters field + // When FormalGenericParameters field is not empty then need to update generic parameters field foreach (var gp in GenericParameters.Keys) { if (GenericParameters[gp] is IGenericTypeParameter specificType) { // Get unfilled type parameter or type parameter that was filled with another type parameter @@ -278,7 +277,7 @@ private void StoreGenericParameters(PythonClassType classType, IGenericTypeParam // class A(Generic[T]): // class B(A[U]) // A has T => U - classType._genericParameters[gp] = genericToSpecificTypes.TryGetValue(specificType, out var v) ? v : null; + specificClassType._genericActualParameters[gp] = genericToSpecificTypes.TryGetValue(specificType, out var v) ? v : null; } } } @@ -300,26 +299,26 @@ private void StoreGenericParameters(PythonClassType classType, IGenericTypeParam /// Dictionary or name (T1) to specific type to populate. private void GetSpecificTypeFromArgumentValue(IGenericType gt, object argumentValue, IDictionary specificTypes) { switch (argumentValue) { - case IPythonDictionary dict when gt.Parameters.Count == 2: + case IPythonDictionary dict when gt.FormalGenericParameters.Count == 2: var keyType = dict.Keys.FirstOrDefault()?.GetPythonType(); var valueType = dict.Values.FirstOrDefault()?.GetPythonType(); if (!keyType.IsUnknown()) { - specificTypes[gt.Parameters[0]] = keyType; + specificTypes[gt.FormalGenericParameters[0]] = keyType; } if (!valueType.IsUnknown()) { - specificTypes[gt.Parameters[1]] = valueType; + specificTypes[gt.FormalGenericParameters[1]] = valueType; } break; - case IPythonIterable iter when gt.TypeId == BuiltinTypeId.List && gt.Parameters.Count == 1: + case IPythonIterable iter when gt.TypeId == BuiltinTypeId.List && gt.FormalGenericParameters.Count == 1: var itemType = iter.GetIterator().Next.GetPythonType(); if (!itemType.IsUnknown()) { - specificTypes[gt.Parameters[0]] = itemType; + specificTypes[gt.FormalGenericParameters[0]] = itemType; } break; - case IPythonCollection coll when gt.TypeId == BuiltinTypeId.Tuple && gt.Parameters.Count >= 1: + case IPythonCollection coll when gt.TypeId == BuiltinTypeId.Tuple && gt.FormalGenericParameters.Count >= 1: var itemTypes = coll.Contents.Select(m => m.GetPythonType()).ToArray(); - for (var i = 0; i < Math.Min(itemTypes.Length, gt.Parameters.Count); i++) { - specificTypes[gt.Parameters[i]] = itemTypes[i]; + for (var i = 0; i < Math.Min(itemTypes.Length, gt.FormalGenericParameters.Count); i++) { + specificTypes[gt.FormalGenericParameters[i]] = itemTypes[i]; } break; } @@ -374,12 +373,27 @@ private void SetClassMembers(PythonClassType classType, IArgumentSet args) { /// Determines if the class is generic. /// A class is generic if it has at least one unfilled generic type parameters or one of its bases is generic /// - public void DecideGeneric() { + private void DecideGeneric() { using (_genericResolutionGuard.Push(this, out var reentered)) { if (!reentered) { - _isGeneric = !Parameters.IsNullOrEmpty() || (Bases?.OfType().Any(g => g.IsGeneric) ?? false); + IsGeneric = !FormalGenericParameters.IsNullOrEmpty() || (Bases?.OfType().Any(g => g.IsGeneric) ?? false); } } } + + private void SetNames() { + // Based on available data, calculate name of generic with parameters, if any, + // as well as qualified name. + if (!_genericActualParameters.IsNullOrEmpty()) { + _nameWithParameters = CodeFormatter.FormatSequence(BaseName, '[', _genericActualParameters.Values); + _qualifiedNameWithParameters = CodeFormatter.FormatSequence(BaseName, '[', _genericActualParameters.Values.Select(v => v.QualifiedName)); + } + } + + private void SetGenericParameters() { + _genericParameters = _genericActualParameters.Values.Distinct().OfType().ToList(); + // Now that parameters are set, check if class is generic + DecideGeneric(); + } } } diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index 8a6acfd6c..47de859fc 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -29,7 +29,7 @@ using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { - [DebuggerDisplay("Class {Name}")] + [DebuggerDisplay("Class {" + nameof(Name) + "}")] internal partial class PythonClassType : PythonType, IPythonClassType, IEquatable { internal enum ClassDocumentationSource { Class, @@ -39,7 +39,6 @@ internal enum ClassDocumentationSource { private static readonly string[] _classMethods = { "mro", "__dict__", @"__weakref__" }; private readonly ReentrancyGuard _memberGuard = new ReentrancyGuard(); - private string _genericName; private List _bases = new List(); private IReadOnlyList _mro; private string _documentation; @@ -61,14 +60,13 @@ public PythonClassType( DeclaringType = declaringType; } + #region IPythonType /// /// If class has generic type parameters, returns that form, e.g 'A[T1, int, ...]', otherwise returns base, e.g 'A' /// - public override string Name => _genericName ?? base.Name; - - #region IPythonType + public override string Name => _nameWithParameters ?? base.Name; + public override string QualifiedName => this.GetQualifiedName(_qualifiedNameWithParameters); public override PythonMemberType MemberType => PythonMemberType.Class; - public IPythonType DeclaringType { get; } public override IEnumerable GetMemberNames() { var names = new HashSet(); @@ -136,8 +134,9 @@ public override string Documentation { if (string.IsNullOrEmpty(_documentation) && Bases != null) { // If still not found, try bases. var o = DeclaringModule.Interpreter.GetBuiltinType(BuiltinTypeId.Object); - _documentation = Bases.FirstOrDefault(b => b != o && !string.IsNullOrEmpty(b?.Documentation)) - ?.Documentation; + _documentation = Bases + .FirstOrDefault(b => b != o && !(b is IGenericClassBase) && !string.IsNullOrEmpty(b?.Documentation))? + .Documentation; DocumentationSource = ClassDocumentationSource.Base; } } @@ -175,7 +174,10 @@ public override IMember Index(IPythonInstance instance, IArgumentSet args) { return fromBases ?? defaultReturn; } + #endregion + #region IPythonClassMember + public IPythonType DeclaringType { get; } #endregion #region IPythonClass @@ -203,12 +205,16 @@ public IReadOnlyList Mro { /// Has the map {T: int, K: str} /// public virtual IReadOnlyDictionary GenericParameters => - _genericParameters ?? EmptyDictionary.Instance; + _genericActualParameters ?? EmptyDictionary.Instance; #endregion internal ClassDocumentationSource DocumentationSource { get; private set; } - internal override void SetDocumentation(string documentation) => _documentation = documentation; + + internal override void SetDocumentation(string documentation) { + _documentation = documentation; + DocumentationSource = ClassDocumentationSource.Class; + } internal void SetBases(IEnumerable bases) { if (_bases.Count > 0) { @@ -234,6 +240,8 @@ internal void SetBases(IEnumerable bases) { } // Invalidate MRO _mro = null; + DecideGeneric(); + if (DeclaringModule is BuiltinsPythonModule) { // TODO: If necessary, we can set __bases__ on builtins when the module is fully analyzed. return; diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index 0ffb34091..13bd73618 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -20,6 +20,7 @@ using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { @@ -30,18 +31,19 @@ namespace Microsoft.Python.Analysis.Types { /// Module making the call. /// Function overload the return value is requested for. /// Call arguments. - /// + /// Location of the call expression. public delegate IMember ReturnValueProvider( IPythonModule declaringModule, IPythonFunctionOverload overload, - IArgumentSet args); + IArgumentSet args, + IndexSpan indexSpan); internal sealed class PythonFunctionOverload : LocatedMember, IPythonFunctionOverload { private readonly string _returnDocumentation; // Allow dynamic function specialization, such as defining return types for builtin // functions that are impossible to scrape and that are missing from stubs. - // Parameters: declaring module, overload for the return value, list of arguments. + // FormalGenericParameters: declaring module, overload for the return value, list of arguments. private ReturnValueProvider _returnValueProvider; // Return value can be an instance or a type info. Consider type(C()) returning @@ -116,7 +118,7 @@ public string GetReturnDocumentation(IPythonType self = null) { public IMember Call(IArgumentSet args, IPythonType self) { if (!_fromAnnotation) { // First try supplied specialization callback. - var rt = _returnValueProvider?.Invoke(args.Eval.Module, this, args); + var rt = _returnValueProvider?.Invoke(args.Eval.Module, this, args, default); if (!rt.IsUnknown()) { return rt; } diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index c6d1067b6..13ee18cfd 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -22,7 +22,7 @@ using Microsoft.Python.Core.Diagnostics; namespace Microsoft.Python.Analysis.Types { - [DebuggerDisplay("{Name}")] + [DebuggerDisplay("{" + nameof(Name) + "}")] internal class PythonType : LocatedMember, IPythonType {//, IEquatable { private readonly object _lock = new object(); private Dictionary _members; diff --git a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs index 5265555fc..f24ffa4a9 100644 --- a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs +++ b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs @@ -80,11 +80,11 @@ public virtual IMember Index(IPythonInstance instance, IArgumentSet args) #endregion #region ILocatedMember - public Location Location => InnerType?.Location ?? default; - public LocationInfo Definition => InnerType?.Definition ?? LocationInfo.Empty; - public IReadOnlyList References => InnerType?.References ?? Array.Empty(); - public void AddReference(Location location) => InnerType?.AddReference(location); - public void RemoveReferences(IPythonModule module) => InnerType?.RemoveReferences(module); + public virtual Location Location => InnerType?.Location ?? default; + public virtual LocationInfo Definition => InnerType?.Definition ?? LocationInfo.Empty; + public virtual IReadOnlyList References => InnerType?.References ?? Array.Empty(); + public virtual void AddReference(Location location) => InnerType?.AddReference(location); + public virtual void RemoveReferences(IPythonModule module) => InnerType?.RemoveReferences(module); #endregion #region IMemberContainer diff --git a/src/Analysis/Ast/Test/ClassesTests.cs b/src/Analysis/Ast/Test/ClassesTests.cs index 2e3008dda..685ce1688 100644 --- a/src/Analysis/Ast/Test/ClassesTests.cs +++ b/src/Analysis/Ast/Test/ClassesTests.cs @@ -574,6 +574,27 @@ class C(B): analysis.Should().HaveClass("C").Which.Should().HaveDocumentation("class C doc"); } + [TestMethod, Priority(0)] + public async Task NoDocFromObject() { + const string code = @" +class A(object): ... +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveClass("A").Which.Documentation.Should().BeNull(); + } + + [TestMethod, Priority(0)] + public async Task NoDocFromGeneric() { + const string code = @" +from typing import Generic, TypeVar + +T = TypeVar('T') +class A(Generic[T]): ... +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Should().HaveClass("A").Which.Documentation.Should().BeNull(); + } + [TestMethod, Priority(0)] public async Task GetAttr() { const string code = @" diff --git a/src/Analysis/Ast/Test/EnumTests.cs b/src/Analysis/Ast/Test/EnumTests.cs new file mode 100644 index 000000000..d2c0af494 --- /dev/null +++ b/src/Analysis/Ast/Test/EnumTests.cs @@ -0,0 +1,48 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class EnumTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + + [TestMethod, Priority(0)] + public async Task BasicEnum() { + const string code = @" +import enum +x = enum.Enum('a', 'b', 'c') +"; + var analysis = await GetAnalysisAsync(code); + var x = analysis.Should().HaveVariable("x").Which; + } + } +} diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 8aec6db86..876886d81 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -15,11 +15,13 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; using FluentAssertions; using FluentAssertions.Execution; using FluentAssertions.Primitives; +using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; @@ -124,7 +126,7 @@ public void HaveSameMembersAs(IMember other) { missingNames.Should().BeEmpty("Subject has missing names: ", missingNames); extraNames.Should().BeEmpty("Subject has extra names: ", extraNames); - foreach (var n in subjectMemberNames) { + foreach (var n in subjectMemberNames.Except(Enumerable.Repeat("__base__", 1))) { var subjectMember = subjectType.GetMember(n); var otherMember = otherContainer.GetMember(n); var subjectMemberType = subjectMember.GetPythonType(); @@ -137,9 +139,23 @@ public void HaveSameMembersAs(IMember other) { subjectMemberType.MemberType.Should().Be(otherMemberType.MemberType); + if(subjectMemberType is IPythonClassType subjectClass) { + var otherClass = otherMemberType as IPythonClassType; + otherClass.Should().NotBeNull(); + + if(subjectClass is IGenericType gt) { + otherClass.Should().BeAssignableTo(); + otherClass.IsGeneric.Should().Be(gt.IsGeneric); + } + + //Debug.Assert(subjectClass.Bases.Count == otherClass.Bases.Count); + subjectClass.Bases.Count.Should().BeGreaterOrEqualTo(otherClass.Bases.Count); + } + if (string.IsNullOrEmpty(subjectMemberType.Documentation)) { otherMemberType.Documentation.Should().BeNullOrEmpty(); } else { + //Debug.Assert(subjectMemberType.Documentation == otherMemberType.Documentation); subjectMemberType.Documentation.Should().Be(otherMemberType.Documentation); } diff --git a/src/Analysis/Ast/Test/FunctionTests.cs b/src/Analysis/Ast/Test/FunctionTests.cs index a8baa20dd..f8c4e8928 100644 --- a/src/Analysis/Ast/Test/FunctionTests.cs +++ b/src/Analysis/Ast/Test/FunctionTests.cs @@ -88,8 +88,8 @@ def f(a, b): "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); var pt = analysis.Should().HaveVariable("pt").Which; - pt.Should().HaveType("Point(x, y)").And.HaveMember("x"); - pt.Should().HaveType("Point(x, y)").And.HaveMember("y"); + pt.Should().HaveType("Point").And.HaveMember("x"); + pt.Should().HaveType("Point").And.HaveMember("y"); } [TestMethod, Priority(0)] diff --git a/src/Analysis/Ast/Test/GenericsTests.cs b/src/Analysis/Ast/Test/GenericsTests.cs index 1d3e17b26..aa62c7cd3 100644 --- a/src/Analysis/Ast/Test/GenericsTests.cs +++ b/src/Analysis/Ast/Test/GenericsTests.cs @@ -389,6 +389,22 @@ from typing import Dict .And.HaveVariable("y").OfType(BuiltinTypeId.Float); } + [TestMethod, Priority(0)] + [Ignore("https://github.com/microsoft/python-language-server/issues/1454")] + public async Task DictPartialParams() { + const string code = @" +from typing import Dict, Generic, TypeVar + +T = TypeVar('T') +class A(Genetic[T], Dict[T, int]) : ... +"; + + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveClass("A") + .Which.Should().HaveBase("Dict[T, int]"); + } + + [TestMethod, Priority(0)] public async Task GenericDictArg() { const string code = @" @@ -869,6 +885,20 @@ def getT(self) -> T: } [TestMethod, Priority(0)] + public async Task GenericBit() { + const string code = @" +from typing import TypeVar, Generic + +_T = TypeVar('_T') + +class A(Generic[_T]): ... +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Diagnostics.Should().BeEmpty(); + var c = analysis.Should().HaveVariable("A").Which.Value.GetPythonType(); + c.IsGeneric.Should().BeTrue(); + } + public async Task GenericClassBaseChain() { const string code = @" from typing import TypeVar, Generic, List diff --git a/src/Analysis/Ast/Test/ImportTests.cs b/src/Analysis/Ast/Test/ImportTests.cs index 5ce08f316..69fd7bfda 100644 --- a/src/Analysis/Ast/Test/ImportTests.cs +++ b/src/Analysis/Ast/Test/ImportTests.cs @@ -223,15 +223,6 @@ public async Task FromFuture() { analysis.Should().HaveFunction("print"); } - [TestMethod, Priority(0)] - public async Task PreferTypeToAny() { - var analysis = await GetAnalysisAsync(@"from TypingConstants import *", PythonVersions.LatestAvailable3X); - analysis.Should().HaveVariable("ONE").Which.Should().HaveType("Any"); - analysis.Should().HaveVariable("TWO").Which.Should().HaveType(BuiltinTypeId.Str); - var a = analysis.Should().HaveClass("A").Which; - a.GetMember("x").Should().HaveType(BuiltinTypeId.Int); - } - [TestMethod, Priority(0)] public async Task StarImportDoesNotOverwriteFunction() { const string code = @" diff --git a/src/Analysis/Ast/Test/TypingTests.cs b/src/Analysis/Ast/Test/TypingTests.cs index 606ecc3c5..2df9a7df7 100644 --- a/src/Analysis/Ast/Test/TypingTests.cs +++ b/src/Analysis/Ast/Test/TypingTests.cs @@ -144,7 +144,6 @@ public async Task Containers() { const string code = @" from typing import * -i : SupportsInt = ... lst_i : List[int] = ... lst_i_0 = lst_i[0] @@ -169,8 +168,7 @@ from typing import * ; var analysis = await GetAnalysisAsync(code); - analysis.Should().HaveVariable("i").OfType(BuiltinTypeId.Int) - .And.HaveVariable("lst_i").OfType("List[int]") + analysis.Should().HaveVariable("lst_i").OfType("List[int]") .And.HaveVariable("lst_i_0").OfType(BuiltinTypeId.Int) .And.HaveVariable("u").OfType("Union[Mapping[int, str], MappingView[str, float], MutableMapping[int, List[str]]]") .And.HaveVariable("dct_s_i").OfType("Mapping[str, int]") @@ -235,9 +233,10 @@ from typing import * "; var analysis = await GetAnalysisAsync(code); - analysis.Should().HaveVariable("n1").OfType("n1(x: int, y: float)") + analysis.Should().HaveVariable("n1").OfType("n1") + .Which.Value.Should().HaveDocumentation("n1(x: int, y: float)"); - .And.HaveVariable("n1_x").OfType(BuiltinTypeId.Int) + analysis.Should().HaveVariable("n1_x").OfType(BuiltinTypeId.Int) .And.HaveVariable("n1_y").OfType(BuiltinTypeId.Float) .And.HaveVariable("n1_0").OfType(BuiltinTypeId.Int) diff --git a/src/Analysis/Core/Impl/Interpreter/ModulePath.cs b/src/Analysis/Core/Impl/Interpreter/ModulePath.cs index 99dba57f1..206b01f21 100644 --- a/src/Analysis/Core/Impl/Interpreter/ModulePath.cs +++ b/src/Analysis/Core/Impl/Interpreter/ModulePath.cs @@ -15,11 +15,7 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text; using System.Text.RegularExpressions; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; diff --git a/src/Caching/Impl/Factories/ClassFactory.cs b/src/Caching/Impl/Factories/ClassFactory.cs deleted file mode 100644 index f0c9be0f6..000000000 --- a/src/Caching/Impl/Factories/ClassFactory.cs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Collections.Generic; -using System.Linq; -using Microsoft.Python.Analysis.Caching.Models; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; -using Microsoft.Python.Parsing; - -namespace Microsoft.Python.Analysis.Caching.Factories { - internal sealed class ClassFactory : FactoryBase { - public ClassFactory(IEnumerable classes, ModuleFactory mf) - : base(classes, mf) { - } - - public override PythonClassType CreateMember(ClassModel cm, IPythonType declaringType) - => new PythonClassType(cm.Name, new Location(ModuleFactory.Module, cm.IndexSpan.ToSpan())); - - protected override void CreateMemberParts(ClassModel cm, PythonClassType cls) { - // In Python 3 exclude object since type creation will add it automatically. - var is3x = ModuleFactory.Module.Interpreter.LanguageVersion.Is3x(); - var bases = cm.Bases.Select(b => is3x && b == "object" ? null : TryCreate(b)).ExcludeDefault().ToArray(); - cls.SetBases(bases); - cls.SetDocumentation(cm.Documentation); - - foreach (var f in cm.Methods) { - cls.AddMember(f.Name, ModuleFactory.FunctionFactory.Construct(f, cls, false), false); - } - foreach (var p in cm.Properties) { - cls.AddMember(p.Name, ModuleFactory.PropertyFactory.Construct(p, cls), false); - } - foreach (var c in cm.InnerClasses) { - cls.AddMember(c.Name, Construct(c, cls, false), false); - } - foreach(var vm in cm.Fields) { - var v = ModuleFactory.VariableFactory.Construct(vm, cls, false); - cls.AddMember(v.Name, v, false); - } - } - } -} diff --git a/src/Caching/Impl/Factories/FactoryBase.cs b/src/Caching/Impl/Factories/FactoryBase.cs deleted file mode 100644 index 2120879a2..000000000 --- a/src/Caching/Impl/Factories/FactoryBase.cs +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Python.Analysis.Caching.Models; -using Microsoft.Python.Analysis.Types; - -namespace Microsoft.Python.Analysis.Caching.Factories { - /// - /// Represents base factory that implements creation of a type - /// from its model (persistent form). - /// - internal abstract class FactoryBase : IDisposable - where TModel : MemberModel - where TMember : IMember { - - private class ModelData { - public TModel Model; - public TMember Member; - } - - private readonly Dictionary _data; - - protected ModuleFactory ModuleFactory { get; } - - protected FactoryBase(IEnumerable models, ModuleFactory mf) { - ModuleFactory = mf; - _data = models.ToDictionary(k => k.Name, v => new ModelData { Model = v }); - } - - public TMember TryCreate(string name, IPythonType declaringType = null) - => _data.TryGetValue(name, out var data) ? Construct(data.Model, declaringType) : default; - - /// - /// Constructs member from its persistent model. - /// - public TMember Construct(TModel cm, IPythonType declaringType = null, bool cached = true) { - TMember m; - - if (!cached) { - m = CreateMember(cm, declaringType); - CreateMemberParts(cm, m); - return m; - } - - var data = _data[cm.Name]; - if (data.Member == null) { - data.Member = CreateMember(data.Model, declaringType); - CreateMemberParts(cm, data.Member); - } - return data.Member; - } - - public virtual void Dispose() => _data.Clear(); - - public abstract TMember CreateMember(TModel model, IPythonType declaringType); - protected virtual void CreateMemberParts(TModel model, TMember member) { } - } -} diff --git a/src/Caching/Impl/Factories/FunctionFactory.cs b/src/Caching/Impl/Factories/FunctionFactory.cs deleted file mode 100644 index 07503bedb..000000000 --- a/src/Caching/Impl/Factories/FunctionFactory.cs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Collections.Generic; -using System.Linq; -using Microsoft.Python.Analysis.Caching.Models; -using Microsoft.Python.Analysis.Types; - -namespace Microsoft.Python.Analysis.Caching.Factories { - internal sealed class FunctionFactory: FactoryBase { - public FunctionFactory(IEnumerable classes, ModuleFactory mf) - : base(classes, mf) { - } - - public override IPythonFunctionType CreateMember(FunctionModel fm, IPythonType declaringType) { - var ft = new PythonFunctionType(fm.Name, new Location(ModuleFactory.Module, fm.IndexSpan.ToSpan()), declaringType, fm.Documentation); - - foreach (var om in fm.Overloads) { - var o = new PythonFunctionOverload(fm.Name, new Location(ModuleFactory.Module, fm.IndexSpan.ToSpan())); - o.SetDocumentation(fm.Documentation); - o.SetReturnValue(ModuleFactory.ConstructMember(om.ReturnType), true); - o.SetParameters(om.Parameters.Select(ConstructParameter).ToArray()); - ft.AddOverload(o); - } - - foreach(var model in fm.Functions) { - var f = CreateMember(model, ft); - if (f != null) { - ft.AddMember(f.Name, f, overwrite: true); - } - } - - foreach (var model in fm.Classes) { - var c = ModuleFactory.ClassFactory.CreateMember(model, ft); - if (c != null) { - ft.AddMember(c.Name, c, overwrite: true); - } - } - - return ft; - } - - private IParameterInfo ConstructParameter(ParameterModel pm) - => new ParameterInfo(pm.Name, ModuleFactory.ConstructType(pm.Type), pm.Kind, ModuleFactory.ConstructMember(pm.DefaultValue)); - } -} diff --git a/src/Caching/Impl/Factories/PropertyFactory.cs b/src/Caching/Impl/Factories/PropertyFactory.cs deleted file mode 100644 index dfbfb51f0..000000000 --- a/src/Caching/Impl/Factories/PropertyFactory.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using Microsoft.Python.Analysis.Caching.Models; -using Microsoft.Python.Analysis.Types; - -namespace Microsoft.Python.Analysis.Caching.Factories { - internal sealed class PropertyFactory { - private readonly ModuleFactory _mf; - - public PropertyFactory(ModuleFactory mf) { - _mf = mf; - } - - public IPythonPropertyType Construct(PropertyModel pm, IPythonClassType cls) { - var prop = new PythonPropertyType(pm.Name, new Location(_mf.Module, pm.IndexSpan.ToSpan()), cls, (pm.Attributes & FunctionAttributes.Abstract) != 0); - prop.SetDocumentation(pm.Documentation); - var o = new PythonFunctionOverload(pm.Name, _mf.DefaultLocation); - o.SetDocumentation(pm.Documentation); // TODO: own documentation? - o.SetReturnValue(_mf.ConstructMember(pm.ReturnType), true); - prop.AddOverload(o); - return prop; - } - } -} diff --git a/src/Caching/Impl/Factories/TypeVarFactory.cs b/src/Caching/Impl/Factories/TypeVarFactory.cs deleted file mode 100644 index 00e38668c..000000000 --- a/src/Caching/Impl/Factories/TypeVarFactory.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Collections.Generic; -using System.Linq; -using Microsoft.Python.Analysis.Caching.Models; -using Microsoft.Python.Analysis.Specializations.Typing.Types; -using Microsoft.Python.Analysis.Types; - -namespace Microsoft.Python.Analysis.Caching.Factories { - internal sealed class TypeVarFactory : FactoryBase { - public TypeVarFactory(IEnumerable models, ModuleFactory mf) - : base(models, mf) { - } - - public override IPythonType CreateMember(TypeVarModel tvm, IPythonType declaringType) - => new GenericTypeParameter(tvm.Name, ModuleFactory.Module, - tvm.Constraints.Select(c => ModuleFactory.ConstructType(c)).ToArray(), - tvm.Bound, tvm.Covariant, tvm.Contravariant, ModuleFactory.DefaultLocation); - } -} diff --git a/src/Caching/Impl/Factories/VariableFactory.cs b/src/Caching/Impl/Factories/VariableFactory.cs deleted file mode 100644 index ad624d183..000000000 --- a/src/Caching/Impl/Factories/VariableFactory.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Collections.Generic; -using Microsoft.Python.Analysis.Caching.Models; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Values; - -namespace Microsoft.Python.Analysis.Caching.Factories { - internal sealed class VariableFactory : FactoryBase { - public VariableFactory(IEnumerable models, ModuleFactory mf) - : base(models, mf) { - } - - public override IVariable CreateMember(VariableModel vm, IPythonType declaringType) { - var m = ModuleFactory.ConstructMember(vm.Value) ?? ModuleFactory.Module.Interpreter.UnknownType; - return new Variable(vm.Name, m, VariableSource.Declaration, new Location(ModuleFactory.Module, vm.IndexSpan?.ToSpan() ?? default)); - } - } -} diff --git a/src/Caching/Impl/GlobalScope.cs b/src/Caching/Impl/GlobalScope.cs index 84d7324fc..ba0c8b4fa 100644 --- a/src/Caching/Impl/GlobalScope.cs +++ b/src/Caching/Impl/GlobalScope.cs @@ -16,48 +16,53 @@ using System; using System.Collections.Generic; using System.Linq; -using Microsoft.Python.Analysis.Caching.Factories; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Caching { internal sealed class GlobalScope : IGlobalScope { private readonly VariableCollection _scopeVariables = new VariableCollection(); + private ModuleModel _model; - public GlobalScope(ModuleModel model, IPythonModule module, IServiceContainer services) { + public GlobalScope(ModuleModel model, IPythonModule module) { + _model = model; Module = module; Name = model.Name; + } - using (var mf = new ModuleFactory(model, module)) { - foreach (var tvm in model.TypeVars) { - var t = mf.TypeVarFactory.Construct(tvm); - _scopeVariables.DeclareVariable(tvm.Name, t, VariableSource.Generic, mf.DefaultLocation); - } - - // Member creation may be non-linear. Consider function A returning instance - // of a class or type info of a function which hasn't been created yet. - // Thus check if member has already been created first. - foreach (var cm in model.Classes) { - var cls = mf.ClassFactory.Construct(cm); - _scopeVariables.DeclareVariable(cm.Name, cls, VariableSource.Declaration, mf.DefaultLocation); - } - - foreach (var fm in model.Functions) { - var ft = mf.FunctionFactory.Construct(fm); - _scopeVariables.DeclareVariable(fm.Name, ft, VariableSource.Declaration, mf.DefaultLocation); - } - - foreach (var vm in model.Variables) { - var v = mf.VariableFactory.Construct(vm); - _scopeVariables.DeclareVariable(vm.Name, v.Value, VariableSource.Declaration, mf.DefaultLocation); - } - // TODO: re-declare __doc__, __name__, etc. + public void ReconstructVariables() { + // Member creation may be non-linear. Consider function A returning instance + // of a class or type info of a function which hasn't been created yet. + // Thus first create members so we can find then, then populate them with content. + var mf = new ModuleFactory(_model, Module); + foreach (var tvm in _model.TypeVars) { + var t = tvm.Construct(mf, null); + _scopeVariables.DeclareVariable(tvm.Name, t, VariableSource.Generic, mf.DefaultLocation); + } + foreach (var ntm in _model.NamedTuples) { + var nt = ntm.Construct(mf, null); + _scopeVariables.DeclareVariable(ntm.Name, nt, VariableSource.Declaration, mf.DefaultLocation); } + foreach (var cm in _model.Classes) { + var cls = cm.Construct(mf, null); + _scopeVariables.DeclareVariable(cm.Name, cls, VariableSource.Declaration, mf.DefaultLocation); + } + foreach (var fm in _model.Functions) { + var ft = fm.Construct(mf, null); + _scopeVariables.DeclareVariable(fm.Name, ft, VariableSource.Declaration, mf.DefaultLocation); + } + foreach (var vm in _model.Variables) { + var v = (IVariable)vm.Construct(mf, null); + _scopeVariables.DeclareVariable(vm.Name, v.Value, VariableSource.Declaration, mf.DefaultLocation); + } + + // TODO: re-declare __doc__, __name__, etc. + _model = null; } + #region IScope public string Name { get; } public ScopeStatement Node => null; public IScope OuterScope => null; @@ -72,5 +77,6 @@ public GlobalScope(ModuleModel model, IPythonModule module, IServiceContainer se public void DeclareVariable(string name, IMember value, VariableSource source, Location location = default) { } public void LinkVariable(string name, IVariable v, Location location) => throw new NotImplementedException() { }; + #endregion } } diff --git a/src/Caching/Impl/Models/CallableModel.cs b/src/Caching/Impl/Models/CallableModel.cs new file mode 100644 index 000000000..ef74613bf --- /dev/null +++ b/src/Caching/Impl/Models/CallableModel.cs @@ -0,0 +1,94 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Utilities; +using Microsoft.Python.Core; +// ReSharper disable MemberCanBeProtected.Global +// ReSharper disable MemberCanBePrivate.Global + +namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] + internal abstract class CallableModel : MemberModel { + public string Documentation { get; set; } + public FunctionAttributes Attributes { get; set; } + public ClassModel[] Classes { get; set; } + public FunctionModel[] Functions { get; set; } + + [NonSerialized] + private readonly ReentrancyGuard _processing = new ReentrancyGuard(); + protected CallableModel() { } // For de-serializer from JSON + + protected CallableModel(IPythonType callable) { + var functions = new List(); + var classes = new List(); + + foreach (var name in callable.GetMemberNames()) { + var m = callable.GetMember(name); + + // Only take members from this class, skip members from bases. + using (_processing.Push(m, out var reentered)) { + if (reentered) { + continue; + } + switch (m) { + case IPythonFunctionType ft1 when ft1.IsLambda(): + break; + case IPythonFunctionType ft2: + functions.Add(new FunctionModel(ft2)); + break; + case IPythonClassType cls: + classes.Add(new ClassModel(cls)); + break; + } + } + } + + Id = callable.Name.GetStableHash(); + Name = callable.Name; + QualifiedName = callable.QualifiedName; + Documentation = callable.Documentation; + Classes = classes.ToArray(); + Functions = functions.ToArray(); + IndexSpan = callable.Location.IndexSpan.ToModel(); + + Attributes = FunctionAttributes.Normal; + if (callable.IsAbstract) { + Attributes |= FunctionAttributes.Abstract; + } + if(callable is IPythonFunctionType ft) { + if(ft.IsClassMethod) { + Attributes |= FunctionAttributes.ClassMethod; + } + if (ft.IsStatic) { + Attributes |= FunctionAttributes.Static; + } + } + //if (callable is IPythonPropertyType p) { + // if (p.IsClassMethod) { + // Attributes |= FunctionAttributes.ClassMethod; + // } + // if (p.IsStatic) { + // Attributes |= FunctionAttributes.Static; + // } + //} + } + + protected override IEnumerable GetMemberModels() => Classes.Concat(Functions); + } +} diff --git a/src/Caching/Impl/Models/ClassModel.cs b/src/Caching/Impl/Models/ClassModel.cs index aa2786670..225a96614 100644 --- a/src/Caching/Impl/Models/ClassModel.cs +++ b/src/Caching/Impl/Models/ClassModel.cs @@ -13,32 +13,47 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Parsing; + +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global +// ReSharper disable MemberCanBePrivate.Global namespace Microsoft.Python.Analysis.Caching.Models { - [DebuggerDisplay("cls:{Name}")] + [Serializable] + [DebuggerDisplay("cls:{" + nameof(Name) + "}")] internal sealed class ClassModel : MemberModel { public string Documentation { get; set; } public string[] Bases { get; set; } + public NamedTupleModel[] NamedTupleBases { get; set; } public FunctionModel[] Methods { get; set; } public PropertyModel[] Properties { get; set; } public VariableModel[] Fields { get; set; } - public string[] GenericParameters { get; set; } - public ClassModel[] InnerClasses { get; set; } + public ClassModel[] Classes { get; set; } - private readonly ReentrancyGuard _processing = new ReentrancyGuard(); + /// + /// FormalGenericParameters of the Generic[...] base class, if any. + /// + public string[] GenericBaseParameters { get; set; } + /// + /// Values assigned to the generic parameters, if any. + /// + public GenericParameterValueModel[] GenericParameterValues { get; set; } - public static ClassModel FromType(IPythonClassType cls) => new ClassModel(cls); + [NonSerialized] private readonly ReentrancyGuard _processing = new ReentrancyGuard(); + [NonSerialized] private PythonClassType _cls; public ClassModel() { } // For de-serializer from JSON - private ClassModel(IPythonClassType cls) { + public ClassModel(IPythonClassType cls) { var methods = new List(); var properties = new List(); var fields = new List(); @@ -57,20 +72,21 @@ private ClassModel(IPythonClassType cls) { if (reentered) { continue; } + switch (m) { case IPythonClassType ct when ct.Name == name: if (!ct.DeclaringModule.Equals(cls.DeclaringModule)) { continue; } - innerClasses.Add(FromType(ct)); + innerClasses.Add(new ClassModel(ct)); break; case IPythonFunctionType ft when ft.IsLambda(): break; case IPythonFunctionType ft when ft.Name == name: - methods.Add(FunctionModel.FromType(ft)); + methods.Add(new FunctionModel(ft)); break; case IPythonPropertyType prop when prop.Name == name: - properties.Add(PropertyModel.FromType(prop)); + properties.Add(new PropertyModel(prop)); break; case IPythonInstance inst: fields.Add(VariableModel.FromInstance(name, inst)); @@ -84,14 +100,108 @@ private ClassModel(IPythonClassType cls) { Name = cls.TypeId == BuiltinTypeId.Ellipsis ? "ellipsis" : cls.Name; Id = Name.GetStableHash(); + QualifiedName = cls.QualifiedName; IndexSpan = cls.Location.IndexSpan.ToModel(); - Documentation = cls.Documentation; - Bases = cls.Bases.OfType().Select(t => t.GetPersistentQualifiedName()).ToArray(); + // Only persist documentation from this class, leave bases or __init__ alone. + Documentation = (cls as PythonClassType)?.DocumentationSource == PythonClassType.ClassDocumentationSource.Class ? cls.Documentation : null; + + + var ntBases = cls.Bases.OfType().ToArray(); + NamedTupleBases = ntBases.Select(b => new NamedTupleModel(b)).ToArray(); + + Bases = cls.Bases.Except(ntBases).Select(t => t.GetPersistentQualifiedName()).ToArray(); Methods = methods.ToArray(); Properties = properties.ToArray(); Fields = fields.ToArray(); - InnerClasses = innerClasses.ToArray(); + Classes = innerClasses.ToArray(); + + if (cls.IsGeneric) { + // Only check immediate bases, i.e. when class itself has Generic[T] base. + var gcp = cls.Bases.OfType().FirstOrDefault(); + GenericBaseParameters = gcp?.TypeParameters.Select(p => p.Name).ToArray(); + } + // If class is generic, we must save its generic base definition + // so on restore we'll be able to re-create the class as generic. + GenericBaseParameters = GenericBaseParameters ?? Array.Empty(); + + GenericParameterValues = cls.GenericParameters + .Select(p => new GenericParameterValueModel { Name = p.Key.Name, Type = p.Value.GetPersistentQualifiedName() }) + .ToArray(); + } + + protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) { + if (_cls != null) { + return _cls; + } + _cls = new PythonClassType(Name, new Location(mf.Module, IndexSpan.ToSpan())); + + var bases = CreateBases(mf); + + _cls.SetBases(bases); + _cls.SetDocumentation(Documentation); + + if (GenericParameterValues.Length > 0) { + _cls.StoreGenericParameters(_cls, + _cls.GenericParameters.Keys.ToArray(), + GenericParameterValues.ToDictionary( + k => _cls.GenericParameters.Keys.First(x => x.Name == k.Name), + v => mf.ConstructType(v.Type) + ) + ); + } + + foreach (var f in Methods) { + var m = f.Construct(mf, _cls); + _cls.AddMember(f.Name, m, false); + } + + foreach (var p in Properties) { + var m = p.Construct(mf, _cls); + _cls.AddMember(p.Name, m, false); + } + + foreach (var c in Classes) { + var m = c.Construct(mf, _cls); + _cls.AddMember(c.Name, m, false); + } + + foreach (var vm in Fields) { + var m = vm.Construct(mf, _cls); + _cls.AddMember(vm.Name, m, false); + } + + return _cls; } + + private IPythonType[] CreateBases(ModuleFactory mf) { + var ntBases = NamedTupleBases.Select(ntb => ntb.Construct(mf, _cls)).OfType().ToArray(); + + var is3x = mf.Module.Interpreter.LanguageVersion.Is3x(); + var basesNames = Bases.Select(b => is3x && b == "object" ? null : b).ExcludeDefault().ToArray(); + var bases = basesNames.Select(mf.ConstructType).ExcludeDefault().Concat(ntBases).ToArray(); + + if (GenericBaseParameters.Length > 0) { + // Generic class. Need to reconstruct generic base so code can then + // create specific types off the generic class. + var genericBase = bases.OfType().FirstOrDefault(b => b.Name == "Generic"); + if (genericBase != null) { + var typeVars = GenericBaseParameters.Select(n => mf.Module.GlobalScope.Variables[n]?.Value).OfType().ToArray(); + Debug.Assert(typeVars.Length > 0, "Class generic type parameters were not defined in the module during restore"); + if (typeVars.Length > 0) { + var genericWithParameters = genericBase.CreateSpecificType(new ArgumentSet(typeVars, null, null)); + if (genericWithParameters != null) { + bases = bases.Except(Enumerable.Repeat(genericBase, 1)).Concat(Enumerable.Repeat(genericWithParameters, 1)).ToArray(); + } + } + } else { + Debug.Fail("Generic class does not have generic base."); + } + } + return bases; + } + + protected override IEnumerable GetMemberModels() + => Classes.Concat(Methods).Concat(Properties).Concat(Fields); } } diff --git a/src/Caching/Impl/Models/FunctionModel.cs b/src/Caching/Impl/Models/FunctionModel.cs index 4608ff719..88da25f78 100644 --- a/src/Caching/Impl/Models/FunctionModel.cs +++ b/src/Caching/Impl/Models/FunctionModel.cs @@ -13,28 +13,59 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; +using System; using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Utilities; -using Microsoft.Python.Core; +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global +// ReSharper disable MemberCanBePrivate.Global namespace Microsoft.Python.Analysis.Caching.Models { - [DebuggerDisplay("f:{Name}")] - internal sealed class FunctionModel : MemberModel { - public string Documentation { get; set; } + [Serializable] + [DebuggerDisplay("f:{" + nameof(Name) + "}")] + internal sealed class FunctionModel : CallableModel { public OverloadModel[] Overloads { get; set; } - public FunctionAttributes Attributes { get; set; } - public ClassModel[] Classes { get; set; } - public FunctionModel[] Functions { get; set; } + public FunctionModel() { } // For de-serializer from JSON + + [NonSerialized] private PythonFunctionType _function; + + public FunctionModel(IPythonFunctionType func) : base(func) { + Overloads = func.Overloads.Select(FromOverload).ToArray(); + } + + protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) { + if (_function != null) { + return _function; + } + _function = new PythonFunctionType(Name, new Location(mf.Module, IndexSpan.ToSpan()), declaringType, Documentation); + + // Create inner functions and classes first since function + // may be returning one of them. + foreach (var model in Functions) { + var f = model.Construct(mf, _function); + _function.AddMember(Name, f, overwrite: true); + } - private readonly ReentrancyGuard _processing = new ReentrancyGuard(); + foreach (var cm in Classes) { + var c = cm.Construct(mf, _function); + _function.AddMember(cm.Name, c, overwrite: true); + } - public static FunctionModel FromType(IPythonFunctionType ft) => new FunctionModel(ft); + foreach (var om in Overloads) { + var o = new PythonFunctionOverload(Name, new Location(mf.Module, IndexSpan.ToSpan())); + o.SetDocumentation(Documentation); + o.SetReturnValue(mf.ConstructMember(om.ReturnType), true); + o.SetParameters(om.Parameters.Select(p => ConstructParameter(mf, p)).ToArray()); + _function.AddOverload(o); + } - private static OverloadModel FromOverload(IPythonFunctionOverload o) { - return new OverloadModel { + return _function; + } + private IParameterInfo ConstructParameter(ModuleFactory mf, ParameterModel pm) + => new ParameterInfo(pm.Name, mf.ConstructType(pm.Type), pm.Kind, mf.ConstructMember(pm.DefaultValue)); + + private static OverloadModel FromOverload(IPythonFunctionOverload o) + => new OverloadModel { Parameters = o.Parameters.Select(p => new ParameterModel { Name = p.Name, Type = p.Type.GetPersistentQualifiedName(), @@ -43,42 +74,5 @@ private static OverloadModel FromOverload(IPythonFunctionOverload o) { }).ToArray(), ReturnType = o.StaticReturnValue.GetPersistentQualifiedName() }; - } - - public FunctionModel() { } // For de-serializer from JSON - - private FunctionModel(IPythonFunctionType func) { - var functions = new List(); - var classes = new List(); - - foreach (var name in func.GetMemberNames()) { - var m = func.GetMember(name); - - // Only take members from this class, skip members from bases. - using (_processing.Push(m, out var reentered)) { - if (reentered) { - continue; - } - switch (m) { - case IPythonFunctionType ft when ft.IsLambda(): - break; - case IPythonFunctionType ft: - functions.Add(FromType(ft)); - break; - case IPythonClassType cls: - classes.Add(ClassModel.FromType(cls)); - break; - } - } - } - - Id = func.Name.GetStableHash(); - Name = func.Name; - IndexSpan = func.Location.IndexSpan.ToModel(); - Documentation = func.Documentation; - Overloads = func.Overloads.Select(FromOverload).ToArray(); - Classes = classes.ToArray(); - Functions = functions.ToArray(); - } } } diff --git a/src/Caching/Impl/Models/GenericParameterValueModel.cs b/src/Caching/Impl/Models/GenericParameterValueModel.cs new file mode 100644 index 000000000..17f0c68f6 --- /dev/null +++ b/src/Caching/Impl/Models/GenericParameterValueModel.cs @@ -0,0 +1,32 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis.Caching.Models { + /// + /// Model for actual values assigned to generic parameters. + /// I.e. if class is based on Generic[T], what is assigned to T. + /// + internal sealed class GenericParameterValueModel { + /// + /// Generic parameter name as defined by TypeVar, such as T. + /// + public string Name { get; set; } + + /// + /// Qualified name of the type assigned to T. + /// + public string Type { get; set; } + } +} diff --git a/src/Caching/Impl/Models/IndexSpanModel.cs b/src/Caching/Impl/Models/IndexSpanModel.cs index e6af970ba..b9ccc45d7 100644 --- a/src/Caching/Impl/Models/IndexSpanModel.cs +++ b/src/Caching/Impl/Models/IndexSpanModel.cs @@ -14,13 +14,15 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Diagnostics; using Microsoft.Python.Core.Text; +// ReSharper disable MemberCanBePrivate.Global namespace Microsoft.Python.Analysis.Caching.Models { + [DebuggerDisplay("{Start}:({Length})")] internal sealed class IndexSpanModel { public int Start { get; set; } public int Length { get; set; } - public IndexSpan ToSpan() => new IndexSpan(Start, Length); } } diff --git a/src/Caching/Impl/Models/MemberModel.cs b/src/Caching/Impl/Models/MemberModel.cs index 253ea6b54..24d512fcf 100644 --- a/src/Caching/Impl/Models/MemberModel.cs +++ b/src/Caching/Impl/Models/MemberModel.cs @@ -13,10 +13,44 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Types; +// ReSharper disable MemberCanBeProtected.Global +// ReSharper disable UnusedAutoPropertyAccessor.Global + namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] internal abstract class MemberModel { + /// + /// Member unique id in the database. + /// public int Id { get; set; } + + /// + /// Member name, such as name of a class. + /// public string Name { get; set; } + + /// + /// Member qualified name within the module, such as A.B.C. + /// + public string QualifiedName { get; set; } + + /// + /// Member location in the module original source code. + /// public IndexSpanModel IndexSpan { get; set; } + + [NonSerialized] + private IMember _member; + + public IMember Construct(ModuleFactory mf, IPythonType declaringType) + => _member ?? (_member = ReConstruct(mf, declaringType)); + protected abstract IMember ReConstruct(ModuleFactory mf, IPythonType declaringType); + + public virtual MemberModel GetModel(string name) => GetMemberModels().FirstOrDefault(m => m.Name == name); + protected virtual IEnumerable GetMemberModels() => Enumerable.Empty(); } } diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs index 7fe06f118..dfe859d35 100644 --- a/src/Caching/Impl/Models/ModuleModel.cs +++ b/src/Caching/Impl/Models/ModuleModel.cs @@ -13,15 +13,18 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; -using Microsoft.Python.Analysis.Caching.Factories; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +// ReSharper disable MemberCanBePrivate.Global namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] internal sealed class ModuleModel : MemberModel { /// /// Module unique id that includes version. @@ -33,6 +36,7 @@ internal sealed class ModuleModel : MemberModel { public VariableModel[] Variables { get; set; } public ClassModel[] Classes { get; set; } public TypeVarModel[] TypeVars { get; set; } + public NamedTupleModel[] NamedTuples { get; set; } /// /// Collection of new line information for conversion of linear spans @@ -45,6 +49,8 @@ internal sealed class ModuleModel : MemberModel { /// public int FileSize { get; set; } + [NonSerialized] private Dictionary _modelCache; + public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceContainer services, AnalysisCachingLevel options) { var uniqueId = analysis.Document.GetUniqueId(services, options); if(uniqueId == null) { @@ -56,6 +62,7 @@ public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceConta var functions = new Dictionary(); var classes = new Dictionary(); var typeVars = new Dictionary(); + var namedTuples = new Dictionary(); // Go directly through variables which names are listed in GetMemberNames // as well as variables that are declarations. @@ -68,9 +75,13 @@ public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceConta if (v.Value is IGenericTypeParameter && !typeVars.ContainsKey(v.Name)) { typeVars[v.Name] = TypeVarModel.FromGeneric(v); + continue; } switch (v.Value) { + case ITypingNamedTupleType nt: + namedTuples[nt.Name] = new NamedTupleModel(nt); + continue; case IPythonFunctionType ft when ft.IsLambda(): // No need to persist lambdas. continue; @@ -92,7 +103,7 @@ public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceConta case IPythonClassType cls when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals(analysis.Document.Stub): if (!classes.ContainsKey(cls.Name)) { - classes[cls.Name] = ClassModel.FromType(cls); + classes[cls.Name] = new ClassModel(cls); continue; } break; @@ -113,6 +124,7 @@ when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals Variables = variables.Values.ToArray(), Classes = classes.Values.ToArray(), TypeVars = typeVars.Values.ToArray(), + NamedTuples = namedTuples.Values.ToArray(), NewLines = analysis.Ast.NewLineLocations.Select(l => new NewLineModel { EndIndex = l.EndIndex, Kind = l.Kind @@ -128,13 +140,28 @@ private static FunctionModel GetFunctionModel(IDocumentAnalysis analysis, IVaria // star import. Their stubs, however, come from 'os' stub. The function then have declaring // module as 'nt' rather than 'os' and 'nt' does not have a stub. In this case use function // model like if function was declared in 'os'. - return FunctionModel.FromType(f); + return new FunctionModel(f); } if (f.DeclaringModule.Equals(analysis.Document) || f.DeclaringModule.Equals(analysis.Document.Stub)) { - return FunctionModel.FromType(f); + return new FunctionModel(f); } return null; } + + protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) => throw new NotImplementedException(); + + public override MemberModel GetModel(string name) { + if (_modelCache == null) { + var models = TypeVars.Concat(NamedTuples).Concat(Classes).Concat(Functions).Concat(Variables); + _modelCache = new Dictionary(); + foreach (var m in models) { + Debug.Assert(!_modelCache.ContainsKey(m.Name)); + _modelCache[m.Name] = m; + } + } + + return _modelCache.TryGetValue(name, out var model) ? model : null; + } } } diff --git a/src/Caching/Impl/Models/NamedTupleModel.cs b/src/Caching/Impl/Models/NamedTupleModel.cs new file mode 100644 index 000000000..5f17059c9 --- /dev/null +++ b/src/Caching/Impl/Models/NamedTupleModel.cs @@ -0,0 +1,54 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Diagnostics; +using System.Linq; +using Microsoft.Python.Analysis.Specializations.Typing; +using Microsoft.Python.Analysis.Specializations.Typing.Types; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; + +namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] + [DebuggerDisplay("n:{" + nameof(Name) + "}")] + internal sealed class NamedTupleModel: MemberModel { + public string[] ItemNames { get; set; } + public string[] ItemTypes { get; set; } + + [NonSerialized] private NamedTupleType _namedTuple; + + public NamedTupleModel() { } // For de-serializer from JSON + + public NamedTupleModel(ITypingNamedTupleType nt) { + Id = nt.Name.GetStableHash(); + Name = nt.Name; + QualifiedName = nt.QualifiedName; + IndexSpan = nt.Location.IndexSpan.ToModel(); + ItemNames = nt.ItemNames.ToArray(); + ItemTypes = nt.ItemTypes.Select(t => t.QualifiedName).ToArray(); + } + + protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) { + if (_namedTuple != null) { + return _namedTuple; + } + + var itemTypes = ItemTypes.Select(mf.ConstructType).ToArray(); + _namedTuple = new NamedTupleType(Name, ItemNames, itemTypes, mf.Module, IndexSpan.ToSpan()); + return _namedTuple; + } + } +} diff --git a/src/Caching/Impl/Models/PropertyModel.cs b/src/Caching/Impl/Models/PropertyModel.cs index 91cb0feef..a5b33e025 100644 --- a/src/Caching/Impl/Models/PropertyModel.cs +++ b/src/Caching/Impl/Models/PropertyModel.cs @@ -13,24 +13,36 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global +// ReSharper disable MemberCanBePrivate.Global namespace Microsoft.Python.Analysis.Caching.Models { - internal sealed class PropertyModel: MemberModel { - public string Documentation { get; set; } + [Serializable] + internal sealed class PropertyModel : CallableModel { public string ReturnType { get; set; } - public FunctionAttributes Attributes { get; set; } + public PropertyModel() { } // For de-serializer from JSON - public static PropertyModel FromType(IPythonPropertyType prop) { - return new PropertyModel { - Id = prop.Name.GetStableHash(), - Name = prop.Name, - IndexSpan = prop.Location.IndexSpan.ToModel(), - Documentation = prop.Documentation, - ReturnType = prop.ReturnType.GetPersistentQualifiedName() - // TODO: attributes. - }; + [NonSerialized] private PythonPropertyType _property; + + public PropertyModel(IPythonPropertyType prop) : base(prop) { + ReturnType = prop.ReturnType.GetPersistentQualifiedName(); + } + + protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) { + if (_property != null) { + return _property; + } + _property = new PythonPropertyType(Name, new Location(mf.Module, IndexSpan.ToSpan()), declaringType, (Attributes & FunctionAttributes.Abstract) != 0); + _property.SetDocumentation(Documentation); + + var o = new PythonFunctionOverload(Name, mf.DefaultLocation); + o.SetDocumentation(Documentation); + o.SetReturnValue(mf.ConstructMember(ReturnType), true); + _property.AddOverload(o); + + return _property; } } } diff --git a/src/Caching/Impl/Models/TypeVarModel.cs b/src/Caching/Impl/Models/TypeVarModel.cs index 0735f1739..5a7bfb2e7 100644 --- a/src/Caching/Impl/Models/TypeVarModel.cs +++ b/src/Caching/Impl/Models/TypeVarModel.cs @@ -13,15 +13,20 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Specializations.Typing; +using Microsoft.Python.Analysis.Specializations.Typing.Types; +using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +// ReSharper disable MemberCanBePrivate.Global namespace Microsoft.Python.Analysis.Caching.Models { - [DebuggerDisplay("TypeVar:{Name}")] - internal sealed class TypeVarModel: MemberModel { + [Serializable] + [DebuggerDisplay("TypeVar:{" + nameof(Name) + "}")] + internal sealed class TypeVarModel : MemberModel { public string[] Constraints { get; set; } public object Bound { get; set; } public object Covariant { get; set; } @@ -32,11 +37,17 @@ public static TypeVarModel FromGeneric(IVariable v) { return new TypeVarModel { Id = g.Name.GetStableHash(), Name = g.Name, + QualifiedName = g.QualifiedName, Constraints = g.Constraints.Select(c => c.GetPersistentQualifiedName()).ToArray(), Bound = g.Bound, Covariant = g.Covariant, Contravariant = g.Contravariant }; } + + protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) + => new GenericTypeParameter(Name, + Constraints.Select(mf.ConstructType).ToArray(), + Bound, Covariant, Contravariant, mf.DefaultLocation); } } diff --git a/src/Caching/Impl/Models/VariableModel.cs b/src/Caching/Impl/Models/VariableModel.cs index e6467d312..c3e2696cf 100644 --- a/src/Caching/Impl/Models/VariableModel.cs +++ b/src/Caching/Impl/Models/VariableModel.cs @@ -13,12 +13,15 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Diagnostics; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +// ReSharper disable MemberCanBePrivate.Global namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] [DebuggerDisplay("v:{Name} = {Value}")] internal sealed class VariableModel: MemberModel { public string Value { get; set; } @@ -26,6 +29,7 @@ internal sealed class VariableModel: MemberModel { public static VariableModel FromVariable(IVariable v) => new VariableModel { Id = v.Name.GetStableHash(), Name = v.Name, + QualifiedName = v.Name, IndexSpan = v.Location.IndexSpan.ToModel(), Value = v.Value.GetPersistentQualifiedName() }; @@ -33,14 +37,21 @@ internal sealed class VariableModel: MemberModel { public static VariableModel FromInstance(string name, IPythonInstance inst) => new VariableModel { Id = name.GetStableHash(), Name = name, + QualifiedName = name, Value = inst.GetPersistentQualifiedName() }; public static VariableModel FromType(string name, IPythonType t) => new VariableModel { Id = name.GetStableHash(), Name = name, + QualifiedName = name, IndexSpan = t.Location.IndexSpan.ToModel(), Value = t.GetPersistentQualifiedName() }; + + protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) { + var m = mf.ConstructMember(Value) ?? mf.Module.Interpreter.UnknownType; + return new Variable(Name, m, VariableSource.Declaration, new Location(mf.Module, IndexSpan?.ToSpan() ?? default)); + } } } diff --git a/src/Caching/Impl/Factories/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs similarity index 69% rename from src/Caching/Impl/Factories/ModuleFactory.cs rename to src/Caching/Impl/ModuleFactory.cs index ddc18af8e..c8bd49041 100644 --- a/src/Caching/Impl/Factories/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -26,37 +26,26 @@ using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; -namespace Microsoft.Python.Analysis.Caching.Factories { - internal sealed class ModuleFactory : IDisposable { +namespace Microsoft.Python.Analysis.Caching { + /// + /// Constructs module from its persistent model. + /// + internal sealed class ModuleFactory { // TODO: better resolve circular references. private readonly ReentrancyGuard _typeReentrancy = new ReentrancyGuard(); private readonly ReentrancyGuard _moduleReentrancy = new ReentrancyGuard(); + private readonly ModuleModel _model; public IPythonModule Module { get; } - public ClassFactory ClassFactory { get; } - public FunctionFactory FunctionFactory { get; } - public PropertyFactory PropertyFactory { get; } - public VariableFactory VariableFactory { get; } - public TypeVarFactory TypeVarFactory { get; } public Location DefaultLocation { get; } public ModuleFactory(ModuleModel model, IPythonModule module) { + _model = model; + Module = module; - ClassFactory = new ClassFactory(model.Classes, this); - FunctionFactory = new FunctionFactory(model.Functions, this); - VariableFactory = new VariableFactory(model.Variables, this); - TypeVarFactory = new TypeVarFactory(model.TypeVars, this); - PropertyFactory = new PropertyFactory(this); DefaultLocation = new Location(Module); } - public void Dispose() { - ClassFactory.Dispose(); - FunctionFactory.Dispose(); - VariableFactory.Dispose(); - TypeVarFactory.Dispose(); - } - public IPythonType ConstructType(string qualifiedName) => ConstructMember(qualifiedName)?.GetPythonType(); public IMember ConstructMember(string qualifiedName) { @@ -65,34 +54,74 @@ public IMember ConstructMember(string qualifiedName) { return null; } - // TODO: better resolve circular references. - using (_typeReentrancy.Push(qualifiedName, out var reentered)) { - if (reentered) { - return null; + // See if member is a module first. + var module = GetModule(parts); + if (module == null) { + return null; + } + + var member = parts.ModuleName == Module.Name + ? GetMemberFromThisModule(parts.MemberNames) + : GetMemberFromModule(module, parts.MemberNames); + + if (parts.ObjectType != ObjectType.Instance) { + return member; + } + + var t = member.GetPythonType() ?? module.Interpreter.UnknownType; + return new PythonInstance(t); + } + + private IMember GetMemberFromThisModule(IReadOnlyList memberNames) { + if (memberNames.Count == 0) { + return null; + } + + // Try from cache first + MemberModel currentModel = _model; + IMember m = null; + IPythonType declaringType = null; + + foreach (var name in memberNames) { + // Check if name has type arguments such as Union[int, str] + // Note that types can be nested like Union[int, Union[A, B]] + var memberName = name; + var typeArgs = GetTypeArguments(memberName, out var typeName); + if (!string.IsNullOrEmpty(typeName) && typeName != name) { + memberName = typeName; } - // See if member is a module first. - var module = GetModule(parts); - if (module == null) { + + if(memberName == "") { return null; } - if (parts.ObjectType == ObjectType.NamedTuple) { - return ConstructNamedTuple(parts.MemberNames[0], module); + var nextModel = currentModel.GetModel(memberName); + Debug.Assert(nextModel != null); + if (nextModel == null) { + return null; } - var member = parts.ModuleName == Module.Name - ? GetMemberFromThisModule(parts.MemberNames) - : GetMemberFromModule(module, parts.MemberNames); - - if (parts.ObjectType != ObjectType.Instance) { - return member; + m = nextModel.Construct(this, declaringType); + Debug.Assert(m != null); + if (m is IGenericType gt && typeArgs.Count > 0) { + m = gt.CreateSpecificType(new ArgumentSet(typeArgs, null, null)); } - var t = member.GetPythonType() ?? module.Interpreter.UnknownType; - return new PythonInstance(t); + currentModel = nextModel; + declaringType = m as IPythonType; + Debug.Assert(declaringType != null); + if (declaringType == null) { + return null; + } } + + return m; } + private IMember GetMemberFromModule(IPythonModule module, IReadOnlyList memberNames) + => memberNames.Count == 0 ? module : GetMember(module, memberNames); + + private IPythonModule GetModule(QualifiedNameParts parts) { if (parts.ModuleName == Module.Name) { return Module; @@ -118,38 +147,8 @@ private IPythonModule GetModule(QualifiedNameParts parts) { } } - private IMember GetMemberFromModule(IPythonModule module, IReadOnlyList memberNames) - => memberNames.Count == 0 ? module : GetMember(module, memberNames); - - private IMember GetBuiltinMember(IBuiltinsPythonModule builtins, string memberName) { - if (memberName.StartsWithOrdinal("__")) { - memberName = memberName.Substring(2, memberName.Length - 4); - } - - switch (memberName) { - case "NoneType": - return builtins.Interpreter.GetBuiltinType(BuiltinTypeId.NoneType); - case "Unknown": - return builtins.Interpreter.UnknownType; - } - return builtins.GetMember(memberName); - } - - private IMember GetMemberFromThisModule(IReadOnlyList memberNames) { - if (memberNames.Count == 0) { - return null; - } - - var name = memberNames[0]; - var root = ClassFactory.TryCreate(name) - ?? (FunctionFactory.TryCreate(name) - ?? (IMember)VariableFactory.TryCreate(name)); - - return GetMember(root, memberNames.Skip(1)); - } - private IMember GetMember(IMember root, IEnumerable memberNames) { - IMember member = root; + var member = root; foreach (var n in memberNames) { var memberName = n; // Check if name has type arguments such as Union[int, str] @@ -168,6 +167,11 @@ private IMember GetMember(IMember root, IEnumerable memberNames) { member = GetBuiltinMember(builtins, memberName) ?? builtins.Interpreter.UnknownType; } else { member = mc?.GetMember(memberName); + // Work around problem that some stubs have incorrectly named tuples. + // For example, in posix.pyi variable for the named tuple is not named as the tuple: + // sched_param = NamedTuple('sched_priority', [('sched_priority', int),]) + member = member ?? (mc as PythonModule)?.GlobalScope.Variables + .FirstOrDefault(v => v.Value is ITypingNamedTupleType nt && nt.Name == memberName); } if (member == null) { @@ -183,6 +187,20 @@ private IMember GetMember(IMember root, IEnumerable memberNames) { return member; } + private IMember GetBuiltinMember(IBuiltinsPythonModule builtins, string memberName) { + if (memberName.StartsWithOrdinal("__")) { + memberName = memberName.Substring(2, memberName.Length - 4); + } + + switch (memberName) { + case "NoneType": + return builtins.Interpreter.GetBuiltinType(BuiltinTypeId.NoneType); + case "Unknown": + return builtins.Interpreter.UnknownType; + } + return builtins.GetMember(memberName); + } + private IReadOnlyList GetTypeArguments(string memberName, out string typeName) { typeName = null; // TODO: better handle generics. @@ -203,7 +221,7 @@ private IReadOnlyList GetTypeArguments(string memberName, out strin if (t == null) { TypeNames.DeconstructQualifiedName(qn, out var parts); typeName = string.Join(".", parts.MemberNames); - t = new GenericTypeParameter(typeName, Module, Array.Empty(), null, null, null, DefaultLocation); + t = new GenericTypeParameter(typeName, Array.Empty(), null, null, null, DefaultLocation); } typeArgs.Add(t); } @@ -212,32 +230,5 @@ private IReadOnlyList GetTypeArguments(string memberName, out strin } return typeArgs; } - - private ITypingNamedTupleType ConstructNamedTuple(string tupleString, IPythonModule module) { - // tuple_name(name: type, name: type, ...) - // time_result(columns: int, lines: int) - var openBraceIndex = tupleString.IndexOf('('); - var closeBraceIndex = tupleString.IndexOf(')'); - var name = tupleString.Substring(0, openBraceIndex); - var argString = tupleString.Substring(openBraceIndex + 1, closeBraceIndex - openBraceIndex - 1); - - var itemNames = new List(); - var itemTypes = new List(); - var start = 0; - - for (var i = 0; i < argString.Length; i++) { - var ch = argString[i]; - if (ch == ':') { - itemNames.Add(argString.Substring(start, i - start).Trim()); - i++; - var paramType = TypeNames.GetTypeName(argString, ref i, ','); - var t = ConstructType(paramType); - itemTypes.Add(t ?? module.Interpreter.UnknownType); - start = i + 1; - } - } - - return new NamedTupleType(name, itemNames, itemTypes, module, module.Interpreter); - } } } diff --git a/src/Caching/Impl/PythonDbModule.cs b/src/Caching/Impl/PythonDbModule.cs index 91a8defa6..addc97053 100644 --- a/src/Caching/Impl/PythonDbModule.cs +++ b/src/Caching/Impl/PythonDbModule.cs @@ -28,7 +28,11 @@ internal sealed class PythonDbModule : SpecializedModule { public PythonDbModule(ModuleModel model, string filePath, IServiceContainer services) : base(model.Name, filePath, services) { - GlobalScope = new GlobalScope(model, this, services); + + var gs = new GlobalScope(model, this); + GlobalScope = gs; + gs.ReconstructVariables(); + Documentation = model.Documentation; _newLines = model.NewLines.Select(nl => new NewLineLocation(nl.EndIndex, nl.Kind)).ToArray(); diff --git a/src/Caching/Impl/QualifiedNameParts.cs b/src/Caching/Impl/QualifiedNameParts.cs index 3bbb431bb..defe456fd 100644 --- a/src/Caching/Impl/QualifiedNameParts.cs +++ b/src/Caching/Impl/QualifiedNameParts.cs @@ -21,8 +21,7 @@ public enum ObjectType { Instance, Module, VariableModule, - BuiltinModule, - NamedTuple + BuiltinModule } internal struct QualifiedNameParts { diff --git a/src/Caching/Impl/TypeNames.cs b/src/Caching/Impl/TypeNames.cs index d60025128..c1e1e7f0e 100644 --- a/src/Caching/Impl/TypeNames.cs +++ b/src/Caching/Impl/TypeNames.cs @@ -32,15 +32,13 @@ public static string GetPersistentQualifiedName(this IMember m) { if (!t.IsUnknown()) { switch (m) { case IPythonInstance _: // constants and strings map here. - return t is ITypingNamedTupleType nt1 ? $"n:{nt1.QualifiedName}" : $"i:{t.QualifiedName}"; + return $"i:{t.QualifiedName}"; case IBuiltinsPythonModule b: return $"b:{b.QualifiedName}"; case PythonVariableModule vm: return $"p:{vm.QualifiedName}"; case IPythonModule mod: return $"m:{mod.QualifiedName}"; - case ITypingNamedTupleType nt2: - return $"n:{nt2.QualifiedName}"; case IPythonType pt when pt.DeclaringModule.ModuleType == ModuleType.Builtins: return $"t:{(pt.TypeId == BuiltinTypeId.Ellipsis ? "ellipsis" : pt.QualifiedName)}"; case IPythonType pt: @@ -82,8 +80,6 @@ private static void GetObjectTypeFromPrefix(string qualifiedName, ref QualifiedN parts.ObjectType = ObjectType.BuiltinModule; } else if (qualifiedName.StartsWith("t:")) { parts.ObjectType = ObjectType.Type; - } else if (qualifiedName.StartsWith("n:")) { - parts.ObjectType = ObjectType.NamedTuple; } else { // Unprefixed name is typically an argument to another type like Union[int, typing:Any] parts.ObjectType = ObjectType.Type; @@ -111,7 +107,7 @@ private static void GetModuleNameAndMembers(string qualifiedName, ref QualifiedN } return; } - + // Extract module name and member names, of any. parts.ModuleName = typeName.Substring(0, moduleSeparatorIndex); var memberNamesOffset = parts.ModuleName.Length + 1; diff --git a/src/Caching/Test/ClassesTests.cs b/src/Caching/Test/ClassesTests.cs index facbf28ff..3aa6c225c 100644 --- a/src/Caching/Test/ClassesTests.cs +++ b/src/Caching/Test/ClassesTests.cs @@ -64,5 +64,91 @@ def methodB2(self): var json = ToJson(model); Baseline.CompareToFile(BaselineFileName, json); } + + [TestMethod, Priority(0)] + public async Task ForwardDeclarations() { + const string code = @" +x = 'str' + +class A: + def methodA1(self): + return B() + + def methodA2(self): + return func() + +class B: + class C: + def methodC(self): + return func() + + def methodB1(self): + def a(): + return 1 + return a + +def func(): + return 1 + +a = B().methodB1() +b = A().methodA1() +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("a").Which.Should().HaveType("a"); + analysis.Should().HaveVariable("b").Which.Should().HaveType("B"); + + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + //var json = ToJson(model); + //Baseline.CompareToFile(BaselineFileName, json); + + using (var dbModule = new PythonDbModule(model, analysis.Document.FilePath, Services)) { + dbModule.Should().HaveSameMembersAs(analysis.Document); + } + } + + [TestMethod, Priority(0)] + public async Task GenericClass() { + const string code = @" +from typing import Generic, TypeVar, Dict + +K = TypeVar('K') +V = TypeVar('V') + +class A(Generic[K, V], Dict[K, V]): + def key(self) -> K: + return K + + def value(self): + return V + +x = A(1, 'a') +"; + var analysis = await GetAnalysisAsync(code); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + //var json = ToJson(model); + //Baseline.CompareToFile(BaselineFileName, json); + + using (var dbModule = new PythonDbModule(model, analysis.Document.FilePath, Services)) { + dbModule.Should().HaveSameMembersAs(analysis.Document); + } + } + + [TestMethod, Priority(0)] + public async Task ClassOwnDocumentation() { + const string code = @" +class A: + '''class A doc''' + +class B(A): + def __init__(self): + '''__init__ doc''' + return +"; + var analysis = await GetAnalysisAsync(code); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + var json = ToJson(model); + // In JSON, class A should have 'class A doc' documentation while B should have none. + Baseline.CompareToFile(BaselineFileName, json); + } } } diff --git a/src/Caching/Test/Files/ClassOwnDocumentation.json b/src/Caching/Test/Files/ClassOwnDocumentation.json new file mode 100644 index 000000000..fb93f52f8 --- /dev/null +++ b/src/Caching/Test/Files/ClassOwnDocumentation.json @@ -0,0 +1,188 @@ +{ + "UniqueId": "module", + "Documentation": "", + "Functions": [], + "Variables": [ + { + "Value": "t:bool", + "Id": -529376420, + "Name": "__debug__", + "QualifiedName": "__debug__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": -1636005055, + "Name": "__doc__", + "QualifiedName": "__doc__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 875442003, + "Name": "__file__", + "QualifiedName": "__file__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 1097116834, + "Name": "__name__", + "QualifiedName": "__name__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 75395663, + "Name": "__package__", + "QualifiedName": "__package__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:list", + "Id": 1154586556, + "Name": "__path__", + "QualifiedName": "__path__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:dict", + "Id": 817929997, + "Name": "__dict__", + "QualifiedName": "__dict__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Classes": [ + { + "Documentation": "class A doc", + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 778, + "Name": "A", + "QualifiedName": "module:A", + "IndexSpan": { + "Start": 8, + "Length": 1 + } + }, + { + "Documentation": null, + "Bases": [ + "t:module:A", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:module:B", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "QualifiedName": "module:B.__init__", + "IndexSpan": { + "Start": 58, + "Length": 8 + } + } + ], + "Properties": [], + "Fields": [], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 779, + "Name": "B", + "QualifiedName": "module:B", + "IndexSpan": { + "Start": 43, + "Length": 1 + } + } + ], + "TypeVars": [], + "NamedTuples": [], + "NewLines": [ + { + "EndIndex": 2, + "Kind": 3 + }, + { + "EndIndex": 12, + "Kind": 3 + }, + { + "EndIndex": 35, + "Kind": 3 + }, + { + "EndIndex": 37, + "Kind": 3 + }, + { + "EndIndex": 50, + "Kind": 3 + }, + { + "EndIndex": 75, + "Kind": 3 + }, + { + "EndIndex": 99, + "Kind": 3 + }, + { + "EndIndex": 115, + "Kind": 3 + } + ], + "FileSize": 115, + "Id": -2131035837, + "Name": "module", + "QualifiedName": null, + "IndexSpan": null +} \ No newline at end of file diff --git a/src/Caching/Test/Files/MemberLocations.json b/src/Caching/Test/Files/MemberLocations.json index a5c60043f..57b4101fd 100644 --- a/src/Caching/Test/Files/MemberLocations.json +++ b/src/Caching/Test/Files/MemberLocations.json @@ -3,7 +3,6 @@ "Documentation": "", "Functions": [ { - "Documentation": null, "Overloads": [ { "Parameters": [ @@ -23,11 +22,13 @@ "ReturnType": null } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": 799444, "Name": "sum", + "QualifiedName": "module:sum", "IndexSpan": { "Start": 19, "Length": 3 @@ -39,6 +40,7 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", + "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, "Length": 0 @@ -48,6 +50,7 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", + "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, "Length": 0 @@ -57,6 +60,7 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", + "QualifiedName": "__file__", "IndexSpan": { "Start": 0, "Length": 0 @@ -66,6 +70,7 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", + "QualifiedName": "__name__", "IndexSpan": { "Start": 0, "Length": 0 @@ -75,6 +80,7 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", + "QualifiedName": "__package__", "IndexSpan": { "Start": 0, "Length": 0 @@ -84,6 +90,7 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", + "QualifiedName": "__path__", "IndexSpan": { "Start": 0, "Length": 0 @@ -93,6 +100,7 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", + "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, "Length": 0 @@ -102,6 +110,7 @@ "Value": "i:str", "Id": 833, "Name": "x", + "QualifiedName": "x", "IndexSpan": { "Start": 2, "Length": 1 @@ -114,9 +123,9 @@ "Bases": [ "t:object" ], + "NamedTupleBases": [], "Methods": [ { - "Documentation": null, "Overloads": [ { "Parameters": [ @@ -130,11 +139,13 @@ "ReturnType": "i:int" } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": 935009768, "Name": "methodB2", + "QualifiedName": "module:B.methodB2", "IndexSpan": { "Start": 253, "Length": 8 @@ -143,11 +154,14 @@ ], "Properties": [ { - "Documentation": "", "ReturnType": "i:int", + "Documentation": "", "Attributes": 0, + "Classes": [], + "Functions": [], "Id": -947452202, "Name": "propertyB", + "QualifiedName": "module:B.propertyB", "IndexSpan": { "Start": 207, "Length": 9 @@ -159,25 +173,25 @@ "Value": "i:int", "Id": 833, "Name": "x", + "QualifiedName": "x", "IndexSpan": null } ], - "GenericParameters": null, - "InnerClasses": [ + "Classes": [ { "Documentation": null, "Bases": [ "t:object" ], + "NamedTupleBases": [], "Methods": [ { - "Documentation": null, "Overloads": [ { "Parameters": [ { "Name": "self", - "Type": "t:module:C", + "Type": "t:module:B.C", "DefaultValue": null, "Kind": 0 } @@ -185,24 +199,25 @@ "ReturnType": null } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": 965872103, "Name": "__init__", + "QualifiedName": "module:B.C.__init__", "IndexSpan": { "Start": 101, "Length": 8 } }, { - "Documentation": null, "Overloads": [ { "Parameters": [ { "Name": "self", - "Type": "t:module:C", + "Type": "t:module:B.C", "DefaultValue": null, "Kind": 0 } @@ -210,11 +225,13 @@ "ReturnType": null } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": -1909501045, "Name": "methodC", + "QualifiedName": "module:B.C.methodC", "IndexSpan": { "Start": 148, "Length": 7 @@ -223,18 +240,23 @@ ], "Properties": [], "Fields": [], - "GenericParameters": null, - "InnerClasses": [], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], "Id": 780, "Name": "C", + "QualifiedName": "module:B.C", "IndexSpan": { "Start": 85, "Length": 1 } } ], + "GenericBaseParameters": [], + "GenericParameterValues": [], "Id": 779, "Name": "B", + "QualifiedName": "module:B", "IndexSpan": { "Start": 57, "Length": 1 @@ -242,6 +264,7 @@ } ], "TypeVars": [], + "NamedTuples": [], "NewLines": [ { "EndIndex": 2, @@ -331,5 +354,6 @@ "FileSize": 288, "Id": -2131035837, "Name": "module", + "QualifiedName": null, "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/NestedClasses.json b/src/Caching/Test/Files/NestedClasses.json index a74431141..bfe0b4731 100644 --- a/src/Caching/Test/Files/NestedClasses.json +++ b/src/Caching/Test/Files/NestedClasses.json @@ -7,6 +7,7 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", + "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, "Length": 0 @@ -16,6 +17,7 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", + "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, "Length": 0 @@ -25,6 +27,7 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", + "QualifiedName": "__file__", "IndexSpan": { "Start": 0, "Length": 0 @@ -34,6 +37,7 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", + "QualifiedName": "__name__", "IndexSpan": { "Start": 0, "Length": 0 @@ -43,6 +47,7 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", + "QualifiedName": "__package__", "IndexSpan": { "Start": 0, "Length": 0 @@ -52,6 +57,7 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", + "QualifiedName": "__path__", "IndexSpan": { "Start": 0, "Length": 0 @@ -61,6 +67,7 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", + "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, "Length": 0 @@ -70,15 +77,17 @@ "Value": "i:str", "Id": 833, "Name": "x", + "QualifiedName": "x", "IndexSpan": { "Start": 2, "Length": 1 } }, { - "Value": "i:module:C", + "Value": "i:module:B.C", "Id": 812, "Name": "c", + "QualifiedName": "c", "IndexSpan": { "Start": 333, "Length": 1 @@ -91,9 +100,9 @@ "Bases": [ "t:object" ], + "NamedTupleBases": [], "Methods": [ { - "Documentation": null, "Overloads": [ { "Parameters": [ @@ -107,11 +116,13 @@ "ReturnType": "i:bool" } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": -1909501047, "Name": "methodA", + "QualifiedName": "module:A.methodA", "IndexSpan": { "Start": 33, "Length": 7 @@ -120,10 +131,12 @@ ], "Properties": [], "Fields": [], - "GenericParameters": null, - "InnerClasses": [], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], "Id": 778, "Name": "A", + "QualifiedName": "module:A", "IndexSpan": { "Start": 21, "Length": 1 @@ -134,9 +147,9 @@ "Bases": [ "t:object" ], + "NamedTupleBases": [], "Methods": [ { - "Documentation": null, "Overloads": [ { "Parameters": [ @@ -147,21 +160,22 @@ "Kind": 0 } ], - "ReturnType": "i:module:C" + "ReturnType": "i:module:B.C" } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": 935009767, "Name": "methodB1", + "QualifiedName": "module:B.methodB1", "IndexSpan": { "Start": 235, "Length": 8 } }, { - "Documentation": null, "Overloads": [ { "Parameters": [ @@ -175,11 +189,13 @@ "ReturnType": "i:int" } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": 935009768, "Name": "methodB2", + "QualifiedName": "module:B.methodB2", "IndexSpan": { "Start": 287, "Length": 8 @@ -192,25 +208,25 @@ "Value": "i:int", "Id": 833, "Name": "x", + "QualifiedName": "x", "IndexSpan": null } ], - "GenericParameters": null, - "InnerClasses": [ + "Classes": [ { "Documentation": null, "Bases": [ "t:object" ], + "NamedTupleBases": [], "Methods": [ { - "Documentation": null, "Overloads": [ { "Parameters": [ { "Name": "self", - "Type": "t:module:C", + "Type": "t:module:B.C", "DefaultValue": null, "Kind": 0 } @@ -218,24 +234,25 @@ "ReturnType": null } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": 965872103, "Name": "__init__", + "QualifiedName": "module:B.C.__init__", "IndexSpan": { "Start": 122, "Length": 8 } }, { - "Documentation": null, "Overloads": [ { "Parameters": [ { "Name": "self", - "Type": "t:module:C", + "Type": "t:module:B.C", "DefaultValue": null, "Kind": 0 } @@ -243,11 +260,13 @@ "ReturnType": "i:bool" } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": -1909501045, "Name": "methodC", + "QualifiedName": "module:B.C.methodC", "IndexSpan": { "Start": 175, "Length": 7 @@ -260,21 +279,27 @@ "Value": "i:int", "Id": 834, "Name": "y", + "QualifiedName": "y", "IndexSpan": null } ], - "GenericParameters": null, - "InnerClasses": [], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], "Id": 780, "Name": "C", + "QualifiedName": "module:B.C", "IndexSpan": { "Start": 106, "Length": 1 } } ], + "GenericBaseParameters": [], + "GenericParameterValues": [], "Id": 779, "Name": "B", + "QualifiedName": "module:B", "IndexSpan": { "Start": 78, "Length": 1 @@ -282,6 +307,7 @@ } ], "TypeVars": [], + "NamedTuples": [], "NewLines": [ { "EndIndex": 2, @@ -379,5 +405,6 @@ "FileSize": 353, "Id": -2131035837, "Name": "module", + "QualifiedName": null, "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/SmokeTest.json b/src/Caching/Test/Files/SmokeTest.json index 00b940866..0f8ebe708 100644 --- a/src/Caching/Test/Files/SmokeTest.json +++ b/src/Caching/Test/Files/SmokeTest.json @@ -3,18 +3,19 @@ "Documentation": "", "Functions": [ { - "Documentation": null, "Overloads": [ { "Parameters": [], "ReturnType": "i:float" } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": 24395611, "Name": "func", + "QualifiedName": "module:func", "IndexSpan": { "Start": 207, "Length": 4 @@ -26,6 +27,7 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", + "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, "Length": 0 @@ -35,6 +37,7 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", + "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, "Length": 0 @@ -44,6 +47,7 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", + "QualifiedName": "__file__", "IndexSpan": { "Start": 0, "Length": 0 @@ -53,6 +57,7 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", + "QualifiedName": "__name__", "IndexSpan": { "Start": 0, "Length": 0 @@ -62,6 +67,7 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", + "QualifiedName": "__package__", "IndexSpan": { "Start": 0, "Length": 0 @@ -71,6 +77,7 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", + "QualifiedName": "__path__", "IndexSpan": { "Start": 0, "Length": 0 @@ -80,6 +87,7 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", + "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, "Length": 0 @@ -89,6 +97,7 @@ "Value": "i:str", "Id": 833, "Name": "x", + "QualifiedName": "x", "IndexSpan": { "Start": 2, "Length": 1 @@ -98,6 +107,7 @@ "Value": "i:module:C", "Id": 812, "Name": "c", + "QualifiedName": "c", "IndexSpan": { "Start": 234, "Length": 1 @@ -110,9 +120,9 @@ "Bases": [ "t:object" ], + "NamedTupleBases": [], "Methods": [ { - "Documentation": null, "Overloads": [ { "Parameters": [ @@ -126,18 +136,19 @@ "ReturnType": null } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": 965872103, "Name": "__init__", + "QualifiedName": "module:C.__init__", "IndexSpan": { "Start": 45, "Length": 8 } }, { - "Documentation": null, "Overloads": [ { "Parameters": [ @@ -151,11 +162,13 @@ "ReturnType": "i:float" } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": -2139806792, "Name": "method", + "QualifiedName": "module:C.method", "IndexSpan": { "Start": 100, "Length": 6 @@ -164,11 +177,14 @@ ], "Properties": [ { - "Documentation": "", "ReturnType": "i:int", + "Documentation": "", "Attributes": 0, + "Classes": [], + "Functions": [], "Id": 24690682, "Name": "prop", + "QualifiedName": "module:C.prop", "IndexSpan": { "Start": 163, "Length": 4 @@ -180,19 +196,23 @@ "Value": "i:int", "Id": 833, "Name": "x", + "QualifiedName": "x", "IndexSpan": null }, { "Value": "i:int", "Id": 834, "Name": "y", + "QualifiedName": "y", "IndexSpan": null } ], - "GenericParameters": null, - "InnerClasses": [], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], "Id": 780, "Name": "C", + "QualifiedName": "module:C", "IndexSpan": { "Start": 21, "Length": 1 @@ -200,6 +220,7 @@ } ], "TypeVars": [], + "NamedTuples": [], "NewLines": [ { "EndIndex": 2, @@ -281,5 +302,6 @@ "FileSize": 243, "Id": -2131035837, "Name": "module", + "QualifiedName": null, "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/VersionHandling2.json b/src/Caching/Test/Files/VersionHandling2.json index 74643926a..f7b1ca27c 100644 --- a/src/Caching/Test/Files/VersionHandling2.json +++ b/src/Caching/Test/Files/VersionHandling2.json @@ -3,7 +3,6 @@ "Documentation": "", "Functions": [ { - "Documentation": null, "Overloads": [ { "Parameters": [ @@ -17,11 +16,13 @@ "ReturnType": null } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": 24395611, "Name": "func", + "QualifiedName": "module:func", "IndexSpan": { "Start": 77, "Length": 4 @@ -33,6 +34,7 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", + "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, "Length": 0 @@ -42,6 +44,7 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", + "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, "Length": 0 @@ -51,6 +54,7 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", + "QualifiedName": "__file__", "IndexSpan": { "Start": 0, "Length": 0 @@ -60,6 +64,7 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", + "QualifiedName": "__name__", "IndexSpan": { "Start": 0, "Length": 0 @@ -69,6 +74,7 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", + "QualifiedName": "__package__", "IndexSpan": { "Start": 0, "Length": 0 @@ -78,6 +84,7 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", + "QualifiedName": "__path__", "IndexSpan": { "Start": 0, "Length": 0 @@ -87,6 +94,7 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", + "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, "Length": 0 @@ -95,6 +103,7 @@ ], "Classes": [], "TypeVars": [], + "NamedTuples": [], "NewLines": [ { "EndIndex": 2, @@ -120,5 +129,6 @@ "FileSize": 91, "Id": -2131035837, "Name": "module", + "QualifiedName": null, "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/VersionHandling3.json b/src/Caching/Test/Files/VersionHandling3.json index 707ef1020..af4e9c068 100644 --- a/src/Caching/Test/Files/VersionHandling3.json +++ b/src/Caching/Test/Files/VersionHandling3.json @@ -3,7 +3,6 @@ "Documentation": "", "Functions": [ { - "Documentation": null, "Overloads": [ { "Parameters": [ @@ -29,11 +28,13 @@ "ReturnType": null } ], + "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": 24395611, "Name": "func", + "QualifiedName": "module:func", "IndexSpan": { "Start": 42, "Length": 4 @@ -45,6 +46,7 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", + "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, "Length": 0 @@ -54,6 +56,7 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", + "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, "Length": 0 @@ -63,6 +66,7 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", + "QualifiedName": "__file__", "IndexSpan": { "Start": 0, "Length": 0 @@ -72,6 +76,7 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", + "QualifiedName": "__name__", "IndexSpan": { "Start": 0, "Length": 0 @@ -81,6 +86,7 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", + "QualifiedName": "__package__", "IndexSpan": { "Start": 0, "Length": 0 @@ -90,6 +96,7 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", + "QualifiedName": "__path__", "IndexSpan": { "Start": 0, "Length": 0 @@ -99,6 +106,7 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", + "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, "Length": 0 @@ -107,6 +115,7 @@ ], "Classes": [], "TypeVars": [], + "NamedTuples": [], "NewLines": [ { "EndIndex": 2, @@ -132,5 +141,6 @@ "FileSize": 91, "Id": -2131035837, "Name": "module", + "QualifiedName": null, "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/LibraryModulesTests.cs b/src/Caching/Test/LibraryModulesTests.cs index cbe98895d..9f9b710ff 100644 --- a/src/Caching/Test/LibraryModulesTests.cs +++ b/src/Caching/Test/LibraryModulesTests.cs @@ -78,7 +78,6 @@ public async Task Builtins() { public Task Crypt() => TestModule("crypt"); [TestMethod, Priority(0)] - [Ignore("_DRMapping type issue. Consider merge of module to stub so OrderedDict resolves to generic from the collections stub.")] public Task Csv() => TestModule("csv"); [TestMethod, Priority(0)] @@ -121,7 +120,6 @@ public async Task Builtins() { public Task Ftplib() => TestModule("ftplib"); [TestMethod, Priority(0)] - [Ignore] public Task Functools() => TestModule("functools"); [TestMethod, Priority(0)] @@ -191,7 +189,7 @@ public async Task Builtins() { public Task Pkgutil() => TestModule("pkgutil"); [TestMethod, Priority(0)] - [Ignore("Specialize Enum. See PlistFormat = enum.Enum('PlistFormat', 'FMT_XML FMT_BINARY', module=__name__)")] + [Ignore("https://github.com/microsoft/python-language-server/issues/1434")] public Task Plistlib() => TestModule("plistlib"); [TestMethod, Priority(0)] diff --git a/src/LanguageServer/Test/GoToDefinitionTests.cs b/src/LanguageServer/Test/GoToDefinitionTests.cs index 35ddf0619..441bcc895 100644 --- a/src/LanguageServer/Test/GoToDefinitionTests.cs +++ b/src/LanguageServer/Test/GoToDefinitionTests.cs @@ -464,5 +464,25 @@ def foo(self): reference.range.Should().Be(2, 8, 2, 16); reference.uri.AbsolutePath.Should().Contain("bar.py"); } + + [TestMethod, Priority(0)] + public async Task NamedTuple() { + const string code = @" +from typing import NamedTuple + +Point = NamedTuple('Point', ['x', 'y']) + +def f(a, b): + return Point(a, b) + +pt = Point(1, 2) +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var ds = new DefinitionSource(Services); + + var reference = ds.FindDefinition(analysis, new SourceLocation(7, 14), out _); + reference.Should().NotBeNull(); + reference.range.Should().Be(3, 0, 3, 5); + } } }