From 2564441b29b52d3df79a563c1e10cee4fb1b02c7 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Feb 2019 20:40:31 +0000 Subject: [PATCH 01/13] Initial cleanup --- mypy/newsemanal/semanal.py | 2 +- mypy/test/hacks.py | 1 - mypy/test/helpers.py | 3 +- test-data/unit/check-newsemanal.test | 43 +++++++++++++++++++++++ test-data/unit/check-type-aliases.test | 47 +++++++++++++++----------- 5 files changed, 74 insertions(+), 22 deletions(-) diff --git a/mypy/newsemanal/semanal.py b/mypy/newsemanal/semanal.py index 2d4d90c3a5487..990beb4407be8 100644 --- a/mypy/newsemanal/semanal.py +++ b/mypy/newsemanal/semanal.py @@ -2224,7 +2224,7 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: s.rvalue.analyzed.column = res.column elif isinstance(s.rvalue, RefExpr): s.rvalue.is_alias_rvalue = True - node.node = TypeAlias(res, node.node.fullname(), s.line, s.column, + node.node = TypeAlias(res, self.qualified_name(lvalue.name), s.line, s.column, alias_tvars=alias_tvars, no_args=no_args) if isinstance(rvalue, RefExpr) and isinstance(rvalue.node, TypeAlias): node.node.normalized = rvalue.node.normalized diff --git a/mypy/test/hacks.py b/mypy/test/hacks.py index 7f7fa910c5de7..03ba23f5899c1 100644 --- a/mypy/test/hacks.py +++ b/mypy/test/hacks.py @@ -45,7 +45,6 @@ 'check-serialize.test', 'check-statements.test', 'check-tuples.test', - 'check-type-aliases.test', 'check-typeddict.test', 'check-typevar-values.test', 'check-unions.test', diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index 1a9e51607a996..f23ab25190d75 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -363,7 +363,8 @@ def parse_options(program_text: str, testcase: DataDrivenTestCase, options.verbosity = testcase.config.getoption('--mypy-verbose') if os.getenv('NEWSEMANAL'): - options.new_semantic_analyzer = True + if not flag_list or '--no-new-semantic-analyzer' not in flag_list: + options.new_semantic_analyzer = True return options diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index aee1f16b54d12..0b4605d68860c 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -975,3 +975,46 @@ class Test: import a def __init__(self) -> None: some_module = self.a + +[case testNewAnalyzerAliasAfterStarImport] +import a +[file a.py] +from typing import Tuple +from b import * + +reveal_type(x) +x: TP +TP = Tuple[int, str] +[file b.py] +from a import x + +reveal_type(x[1]) + +[case testNewAnalyzerAliasToNotReadyClass] +import a +[file a.py] +from b import B + +x: A +A = B +[file b.py] +from typing import List +from a import x + +class B(List[B]): pass + +reveal_type(x) +reveal_type(x[0]) +reveal_type(x[0][0]) +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyClass2] +from typing import List + +x: A + +class A(List[B]): pass +B = A + +reveal_type(x[0][0]) +[builtins fixtures/list.pyi] diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index bc5bf9f2c4cc6..0bd2365d8c8eb 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -178,26 +178,30 @@ Alias = Tuple[int, T] [out] [case testRecursiveAliasesErrors1] +# flags: --no-new-semantic-analyzer +# Recursive aliases are not supported yet. from typing import Type, Callable, Union A = Union[A, int] B = Callable[[B], int] C = Type[C] [out] -main:3: error: Recursive types not fully supported yet, nested types replaced with "Any" -main:4: error: Recursive types not fully supported yet, nested types replaced with "Any" main:5: error: Recursive types not fully supported yet, nested types replaced with "Any" +main:6: error: Recursive types not fully supported yet, nested types replaced with "Any" +main:7: error: Recursive types not fully supported yet, nested types replaced with "Any" [case testRecursiveAliasesErrors2] +# flags: --no-new-semantic-analyzer +# Recursive aliases are not supported yet. from typing import Type, Callable, Union A = Union[B, int] B = Callable[[C], int] C = Type[A] [out] -main:3: error: Recursive types not fully supported yet, nested types replaced with "Any" -main:4: error: Recursive types not fully supported yet, nested types replaced with "Any" main:5: error: Recursive types not fully supported yet, nested types replaced with "Any" +main:6: error: Recursive types not fully supported yet, nested types replaced with "Any" +main:7: error: Recursive types not fully supported yet, nested types replaced with "Any" [case testDoubleForwardAlias] from typing import List @@ -219,6 +223,8 @@ reveal_type(x[0].x) # E: Revealed type is 'builtins.str' [out] [case testJSONAliasApproximation] +# flags: --no-new-semantic-analyzer +# Recursive aliases are not supported yet. from typing import List, Union, Dict x: JSON JSON = Union[int, str, List[JSON], Dict[str, JSON]] # type: ignore @@ -228,35 +234,39 @@ if isinstance(x, list): [builtins fixtures/isinstancelist.pyi] [out] -[case testProhibitedForwardRefToTypeVar] +[case testForwardRefToTypeVar] +# flags: --new-semantic-analyzer from typing import TypeVar, List - -a: List[T] - +reveal_type(a) # E: Revealed type is 'builtins.list[builtins.int]' +a: A[int] +A = List[T] T = TypeVar('T') [builtins fixtures/list.pyi] [out] -main:3: error: Invalid type "__main__.T" -main:3: note: Forward references to type variables are prohibited -[case testUnsupportedForwardRef] +[case testFunctionForwardRefAlias] +# flags: --new-semantic-analyzer from typing import List, TypeVar T = TypeVar('T') -def f(x: T) -> None: - y: A[T] # E: Unsupported forward reference to "A" +def f(x: T) -> List[T]: + y: A[T] + reveal_type(y) # E: Revealed type is 'builtins.list[T`-1]' + return [x] + y A = List[T] [builtins fixtures/list.pyi] [out] -[case testUnsupportedForwardRef2] +[case testFunctionForwardRefAlias2] +# flags: --new-semantic-analyzer from typing import List, TypeVar def f() -> None: X = List[int] - x: A[X] # E: Unsupported forward reference to "A" + x: A[X] + reveal_type(x) # E: Revealed type is 'builtins.list[builtins.list[builtins.int]]' T = TypeVar('T') A = List[T] @@ -468,6 +478,7 @@ class Parameter: [case testAliasInImportCycle3] # cmd: mypy -m t t2 +# flags: --new-semantic-analyzer [file t.py] MYPY = False if MYPY: @@ -475,10 +486,8 @@ if MYPY: x: A reveal_type(x) # E: Revealed type is 't2.D' -# Unfortunately runtime part doesn't work yet, see docstring in SemanticAnalyzerPass3.update_imported_vars() -reveal_type(A) # E: Revealed type is 'Any' \ - # E: Cannot determine type of 'A' -A() # E: Cannot determine type of 'A' +reveal_type(A) # E: Revealed type is 'def () -> t2.D' +A() [file t2.py] import t class D: pass From 8f230f2a86eab1907ac31c1ee5b4de8a13b10dd0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Feb 2019 21:14:02 +0000 Subject: [PATCH 02/13] Some fixes ideas --- mypy/newsemanal/semanal.py | 15 +++++++++++++-- mypy/newsemanal/semanal_main.py | 2 +- mypy/newsemanal/typeanal.py | 3 ++- test-data/unit/check-newsemanal.test | 6 ++---- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/mypy/newsemanal/semanal.py b/mypy/newsemanal/semanal.py index 990beb4407be8..47ef4e2d559e5 100644 --- a/mypy/newsemanal/semanal.py +++ b/mypy/newsemanal/semanal.py @@ -1886,16 +1886,24 @@ def add_type_alias_deps(self, aliases_used: Iterable[str], target = self.scope.current_target() self.cur_mod_node.alias_deps[target].update(aliases_used) + def can_be_an_alias(self, rv: Expression) -> bool: + if isinstance(rv, NameExpr): + n = self.lookup(rv.name, rv) + if n and isinstance(n.node, PlaceholderNode) and n.node.becomes_typeinfo: + return True + return False + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: s.is_final_def = self.unwrap_final(s) tag = self.track_incomplete_refs() s.rvalue.accept(self) - if self.found_incomplete_ref(tag): + is_alias = self.can_be_an_alias(s.rvalue) + if self.found_incomplete_ref(tag) or is_alias: # Initializer couldn't be fully analyzed. Defer the current node and give up. # Make sure that if we skip the definition of some local names, they can't be # added later in this scope, since an earlier definition should take precedence. for expr in names_modified_by_assignment(s): - self.mark_incomplete(expr.name, expr) + self.mark_incomplete(expr.name, expr, becomes_typeinfo=is_alias) return if self.analyze_namedtuple_assign(s): return @@ -2065,6 +2073,9 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: analyzed = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal) if analyzed is None: return + if isinstance(analyzed, UnboundType) and not self.final_iteration: + self.defer() + return s.type = analyzed if (self.type and self.type.is_protocol and isinstance(lvalue, NameExpr) and isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs): diff --git a/mypy/newsemanal/semanal_main.py b/mypy/newsemanal/semanal_main.py index 398869d4f974e..62971ba6c750f 100644 --- a/mypy/newsemanal/semanal_main.py +++ b/mypy/newsemanal/semanal_main.py @@ -135,7 +135,7 @@ def process_top_level_function(analyzer: 'NewSemanticAnalyzer', # OK, this is one last pass, now missing names will be reported. more_iterations = False analyzer.incomplete_namespaces.discard(module) - deferred, incomplete = semantic_analyze_target(target, state, node, active_type, False) + deferred, incomplete = semantic_analyze_target(target, state, node, active_type, True) # After semantic analysis is done, discard local namespaces # to avoid memory hoarding. diff --git a/mypy/newsemanal/typeanal.py b/mypy/newsemanal/typeanal.py index c64406265a726..6bb71f305b0df 100644 --- a/mypy/newsemanal/typeanal.py +++ b/mypy/newsemanal/typeanal.py @@ -421,7 +421,8 @@ def analyze_unbound_type_without_type_info(self, t: UnboundType, sym: SymbolTabl if self.allow_unbound_tvars and unbound_tvar and not self.third_pass: return t # None of the above options worked, we give up. - self.fail('Invalid type "{}"'.format(name), t) + if self.api.final_iteration: + self.fail('Invalid type "{}"'.format(name), t) if self.third_pass and isinstance(sym.node, TypeVarExpr): self.note_func("Forward references to type variables are prohibited", t) return AnyType(TypeOfAny.from_error) diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index 0b4605d68860c..0f9b2d4ec8684 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -1003,9 +1003,7 @@ from a import x class B(List[B]): pass -reveal_type(x) -reveal_type(x[0]) -reveal_type(x[0][0]) +reveal_type(x[0][0]) # E: Revealed type is 'b.B*' [builtins fixtures/list.pyi] [case testNewAnalyzerAliasToNotReadyClass2] @@ -1016,5 +1014,5 @@ x: A class A(List[B]): pass B = A -reveal_type(x[0][0]) +reveal_type(x[0][0]) # E: Revealed type is '__main__.A*' [builtins fixtures/list.pyi] From 3ea427f96488f175ac69dbe759e82d25b8f83932 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Feb 2019 17:41:47 +0000 Subject: [PATCH 03/13] Fix self-check and function logic; add a test --- mypy/newsemanal/semanal.py | 4 +++- mypy/newsemanal/semanal_main.py | 2 +- mypy/newsemanal/semanal_shared.py | 5 +++++ mypy/newsemanal/typeanal.py | 4 +++- test-data/unit/check-newsemanal.test | 11 +++++++++++ 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/mypy/newsemanal/semanal.py b/mypy/newsemanal/semanal.py index 47ef4e2d559e5..23cf36c98a8a3 100644 --- a/mypy/newsemanal/semanal.py +++ b/mypy/newsemanal/semanal.py @@ -213,6 +213,9 @@ class NewSemanticAnalyzer(NodeVisitor[None], # Stack of functions being analyzed function_stack = None # type: List[FuncItem] + # Is this the final iteration of semantic analysis? + final_iteration = False + loop_depth = 0 # Depth of breakable loops cur_mod_id = '' # Current module id (or None) (phase 2) is_stub_file = False # Are we analyzing a stub file? @@ -2074,7 +2077,6 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: if analyzed is None: return if isinstance(analyzed, UnboundType) and not self.final_iteration: - self.defer() return s.type = analyzed if (self.type and self.type.is_protocol and isinstance(lvalue, NameExpr) and diff --git a/mypy/newsemanal/semanal_main.py b/mypy/newsemanal/semanal_main.py index 62971ba6c750f..db72148259450 100644 --- a/mypy/newsemanal/semanal_main.py +++ b/mypy/newsemanal/semanal_main.py @@ -135,7 +135,7 @@ def process_top_level_function(analyzer: 'NewSemanticAnalyzer', # OK, this is one last pass, now missing names will be reported. more_iterations = False analyzer.incomplete_namespaces.discard(module) - deferred, incomplete = semantic_analyze_target(target, state, node, active_type, True) + deferred, incomplete = semantic_analyze_target(target, state, node, active_type, not more_iterations) # After semantic analysis is done, discard local namespaces # to avoid memory hoarding. diff --git a/mypy/newsemanal/semanal_shared.py b/mypy/newsemanal/semanal_shared.py index 39279eee998af..f2f1416c0e9b6 100644 --- a/mypy/newsemanal/semanal_shared.py +++ b/mypy/newsemanal/semanal_shared.py @@ -73,6 +73,11 @@ def is_incomplete_namespace(self, fullname: str) -> bool: """Is a module or class namespace potentially missing some definitions?""" raise NotImplementedError + @abstractproperty + def final_iteration(self) -> bool: + """Is this the final iteration of semantic analysis?""" + raise NotImplementedError + @trait class SemanticAnalyzerInterface(SemanticAnalyzerCoreInterface): diff --git a/mypy/newsemanal/typeanal.py b/mypy/newsemanal/typeanal.py index 6bb71f305b0df..a03c9d16789a0 100644 --- a/mypy/newsemanal/typeanal.py +++ b/mypy/newsemanal/typeanal.py @@ -423,6 +423,8 @@ def analyze_unbound_type_without_type_info(self, t: UnboundType, sym: SymbolTabl # None of the above options worked, we give up. if self.api.final_iteration: self.fail('Invalid type "{}"'.format(name), t) + else: + self.api.defer() if self.third_pass and isinstance(sym.node, TypeVarExpr): self.note_func("Forward references to type variables are prohibited", t) return AnyType(TypeOfAny.from_error) @@ -693,7 +695,7 @@ def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> Optional[L # We report an error in only the first two cases. In the third case, we assume # some other region of the code has already reported a more relevant error. # - # TODO: Once we start adding support for enums, make sure we reprt a custom + # TODO: Once we start adding support for enums, make sure we report a custom # error for case 2 as well. if arg.type_of_any != TypeOfAny.from_error: self.fail('Parameter {} of Literal[...] cannot be of type "Any"'.format(idx), ctx) diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index 0f9b2d4ec8684..d3a823e2c80aa 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -1016,3 +1016,14 @@ B = A reveal_type(x[0][0]) # E: Revealed type is '__main__.A*' [builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyClass3] +from typing import List + +x: B +B = A +A = C +class C(List[B]): pass + +reveal_type(x[0][0]) # E: Revealed type is '__main__.C*' +[builtins fixtures/list.pyi] From c6d3818747c9fd1f4396fb6b62491295ecfdc3c6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Feb 2019 19:50:33 +0000 Subject: [PATCH 04/13] Fix crash in mypy self-check --- mypy/newsemanal/semanal_main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/newsemanal/semanal_main.py b/mypy/newsemanal/semanal_main.py index db72148259450..a08c15ab9fffe 100644 --- a/mypy/newsemanal/semanal_main.py +++ b/mypy/newsemanal/semanal_main.py @@ -137,6 +137,7 @@ def process_top_level_function(analyzer: 'NewSemanticAnalyzer', analyzer.incomplete_namespaces.discard(module) deferred, incomplete = semantic_analyze_target(target, state, node, active_type, not more_iterations) + analyzer.incomplete_namespaces.discard(module) # After semantic analysis is done, discard local namespaces # to avoid memory hoarding. analyzer.saved_locals.clear() From bddb52b1b2436351f07bc88260ae44d02214fa63 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Feb 2019 19:52:19 +0000 Subject: [PATCH 05/13] Skip two tests in new analyzer --- test-data/unit/check-type-aliases.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 0bd2365d8c8eb..bada7486bd6e8 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -588,6 +588,7 @@ def foo(x: Bogus[int]) -> None: [builtins fixtures/dict.pyi] [case testOverrideByIdemAliasCorrectType] +# flags: --no-new-semantic-analyzer C = C class C: # type: ignore pass @@ -614,6 +615,7 @@ class C: [out] [case testConditionalExceptionAlias] +# flags: --no-new-semantic-analyzer try: E = E except BaseException: From b0f6db5ef4df5de797e0744a8032ff587f1f1557 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Feb 2019 20:30:15 +0000 Subject: [PATCH 06/13] Bake-out one of the tests --- test-data/unit/check-newsemanal.test | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index d3a823e2c80aa..aecb8d6e01784 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -976,20 +976,6 @@ class Test: def __init__(self) -> None: some_module = self.a -[case testNewAnalyzerAliasAfterStarImport] -import a -[file a.py] -from typing import Tuple -from b import * - -reveal_type(x) -x: TP -TP = Tuple[int, str] -[file b.py] -from a import x - -reveal_type(x[1]) - [case testNewAnalyzerAliasToNotReadyClass] import a [file a.py] From 39354d7ec5bcdf45e29cd962dd0d10289976c0bb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Feb 2019 20:32:09 +0000 Subject: [PATCH 07/13] Fix lint --- mypy/newsemanal/semanal_main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/newsemanal/semanal_main.py b/mypy/newsemanal/semanal_main.py index a08c15ab9fffe..c56b9936148c5 100644 --- a/mypy/newsemanal/semanal_main.py +++ b/mypy/newsemanal/semanal_main.py @@ -135,7 +135,8 @@ def process_top_level_function(analyzer: 'NewSemanticAnalyzer', # OK, this is one last pass, now missing names will be reported. more_iterations = False analyzer.incomplete_namespaces.discard(module) - deferred, incomplete = semantic_analyze_target(target, state, node, active_type, not more_iterations) + deferred, incomplete = semantic_analyze_target(target, state, node, active_type, + not more_iterations) analyzer.incomplete_namespaces.discard(module) # After semantic analysis is done, discard local namespaces From 84c70b16f318d410e633102b1b398c0e88e84621 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Feb 2019 21:27:44 +0000 Subject: [PATCH 08/13] Extend support to MemberExpr (nested classes); and IndexExpr (generics) --- mypy/newsemanal/semanal.py | 10 +++- test-data/unit/check-newsemanal.test | 78 ++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/mypy/newsemanal/semanal.py b/mypy/newsemanal/semanal.py index a1ab94f4e5434..657ee0932737f 100644 --- a/mypy/newsemanal/semanal.py +++ b/mypy/newsemanal/semanal.py @@ -1886,10 +1886,17 @@ def add_type_alias_deps(self, aliases_used: Iterable[str], self.cur_mod_node.alias_deps[target].update(aliases_used) def can_be_an_alias(self, rv: Expression) -> bool: + if isinstance(rv, IndexExpr) and isinstance(rv.base, RefExpr): + return self.can_be_an_alias(rv.base) if isinstance(rv, NameExpr): n = self.lookup(rv.name, rv) if n and isinstance(n.node, PlaceholderNode) and n.node.becomes_typeinfo: return True + elif isinstance(rv, MemberExpr): + fname = get_member_expr_fullname(rv) + n = self.lookup_qualified(fname, rv) + if n and isinstance(n.node, PlaceholderNode) and n.node.becomes_typeinfo: + return True return False def visit_assignment_stmt(self, s: AssignmentStmt) -> None: @@ -2202,8 +2209,9 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: # annotations (see the second rule). return rvalue = s.rvalue + tag = self.track_incomplete_refs() res, alias_tvars, depends_on, qualified_tvars = self.analyze_alias(rvalue) - if not res: + if not res or self.found_incomplete_ref(tag): return if (isinstance(res, Instance) and res.type.name() == lvalue.name and res.type.module_name == self.cur_mod_id): diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index f2867c5054bb7..e42f80f2f9610 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -1092,3 +1092,81 @@ class C(List[B]): pass reveal_type(x[0][0]) # E: Revealed type is '__main__.C*' [builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyNestedClass] +import a +[file a.py] +from b import Out + +x: A +A = Out.B +[file b.py] +from typing import List +from a import x + +class Out: + class B(List[B]): pass + +reveal_type(x[0][0]) # E: Revealed type is 'b.Out.B*' +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyNestedClass2] +from typing import List + +x: Out.A + +class Out: + class A(List[B]): pass +B = Out.A + +reveal_type(x[0][0]) # E: Revealed type is '__main__.Out.A*' +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyClassGeneric] +import a +[file a.py] +from typing import Tuple +from b import B, T + +x: A[int] +A = B[Tuple[T, T]] +[file b.py] +from typing import List, Generic, TypeVar +from a import x + +class B(List[B], Generic[T]): pass +T = TypeVar('T') +reveal_type(x) # E: Revealed type is 'b.B[Tuple[builtins.int, builtins.int]]' +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyClassInGeneric] +import a +[file a.py] +from typing import Tuple +from b import B + +x: A +A = Tuple[B, B] +[file b.py] +from typing import List +from a import x + +class B(List[B]): pass + +reveal_type(x) # E: Revealed type is 'Tuple[b.B, b.B]' +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyClassDoubleGeneric] +from typing import List, TypeVar, Union + +T = TypeVar('T') + +x: B[int] +B = A[List[T]] +A = Union[int, T] +class C(List[B[int]]): pass + +reveal_type(x) # E: Revealed type is 'Union[builtins.int, builtins.list[builtins.int]]' +reveal_type(y[0]) # E: Revealed type is 'Union[builtins.int, builtins.list[builtins.int]]' +y: C +[builtins fixtures/list.pyi] From c958e7b646433e5d2c59fe1915fd5c0222822467 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Feb 2019 21:35:30 +0000 Subject: [PATCH 09/13] Fix self-check --- mypy/newsemanal/semanal.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mypy/newsemanal/semanal.py b/mypy/newsemanal/semanal.py index 657ee0932737f..608056a294a88 100644 --- a/mypy/newsemanal/semanal.py +++ b/mypy/newsemanal/semanal.py @@ -1894,9 +1894,10 @@ def can_be_an_alias(self, rv: Expression) -> bool: return True elif isinstance(rv, MemberExpr): fname = get_member_expr_fullname(rv) - n = self.lookup_qualified(fname, rv) - if n and isinstance(n.node, PlaceholderNode) and n.node.becomes_typeinfo: - return True + if fname: + n = self.lookup_qualified(fname, rv) + if n and isinstance(n.node, PlaceholderNode) and n.node.becomes_typeinfo: + return True return False def visit_assignment_stmt(self, s: AssignmentStmt) -> None: From e45c7ebdd4c6644ce1b8bab7ad5c93eb4500f31e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Feb 2019 23:34:54 +0000 Subject: [PATCH 10/13] An attempt at fixing string references --- mypy/newsemanal/semanal.py | 3 +-- mypy/newsemanal/typeanal.py | 2 +- test-data/unit/check-newsemanal.test | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/mypy/newsemanal/semanal.py b/mypy/newsemanal/semanal.py index 608056a294a88..f57cf9ef95856 100644 --- a/mypy/newsemanal/semanal.py +++ b/mypy/newsemanal/semanal.py @@ -1887,6 +1887,7 @@ def add_type_alias_deps(self, aliases_used: Iterable[str], def can_be_an_alias(self, rv: Expression) -> bool: if isinstance(rv, IndexExpr) and isinstance(rv.base, RefExpr): + self.analyze_alias(rv) return self.can_be_an_alias(rv.base) if isinstance(rv, NameExpr): n = self.lookup(rv.name, rv) @@ -2087,8 +2088,6 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: analyzed = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal) if analyzed is None: return - if isinstance(analyzed, UnboundType) and not self.final_iteration: - return s.type = analyzed if (self.type and self.type.is_protocol and isinstance(lvalue, NameExpr) and isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs): diff --git a/mypy/newsemanal/typeanal.py b/mypy/newsemanal/typeanal.py index a03c9d16789a0..9c71a17275fac 100644 --- a/mypy/newsemanal/typeanal.py +++ b/mypy/newsemanal/typeanal.py @@ -121,7 +121,7 @@ def analyze_type_alias(node: Expression, api.fail('Invalid type alias', node) return None analyzer = TypeAnalyser(api, tvar_scope, plugin, options, is_typeshed_stub, - allow_unnormalized=allow_unnormalized, defining_alias=True) + allow_unnormalized=allow_unnormalized, defining_alias=True, allow_placeholder=True) analyzer.in_dynamic_func = in_dynamic_func analyzer.global_scope = global_scope res = type.accept(analyzer) diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index e42f80f2f9610..75f2003b969e7 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -1170,3 +1170,21 @@ reveal_type(x) # E: Revealed type is 'Union[builtins.int, builtins.list[builtin reveal_type(y[0]) # E: Revealed type is 'Union[builtins.int, builtins.list[builtins.int]]' y: C [builtins fixtures/list.pyi] + +[case testNewAnalyzerForwardAliasFromUnion] +from typing import Union, List + +A = Union['B', 'C'] + +class D: + x: List[A] + + def test(self) -> None: + reveal_type(self.x[0].y) # E: Revealed type is 'builtins.int' + + +class B: + y: int +class C: + y: int +[builtins fixtures/list.pyi] From 20c85a14ccd7d7ebb0a453efcb0af23cbb21060c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 14 Feb 2019 12:53:39 +0000 Subject: [PATCH 11/13] A cleaner fix --- mypy/newsemanal/semanal.py | 23 +++++++++++++++++------ mypy/newsemanal/typeanal.py | 4 +++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/mypy/newsemanal/semanal.py b/mypy/newsemanal/semanal.py index f57cf9ef95856..cfe094de37ca0 100644 --- a/mypy/newsemanal/semanal.py +++ b/mypy/newsemanal/semanal.py @@ -1885,10 +1885,14 @@ def add_type_alias_deps(self, aliases_used: Iterable[str], target = self.scope.current_target() self.cur_mod_node.alias_deps[target].update(aliases_used) - def can_be_an_alias(self, rv: Expression) -> bool: + def is_not_ready_alias(self, rv: Expression) -> bool: + """Does this expression look like a not ready type alias? + + This includes 'Ref' and 'Ref[Arg1, Arg2, ...]', where 'Ref' + refers to a PlaceholderNode with becomes_typeinfo=True. + """ if isinstance(rv, IndexExpr) and isinstance(rv.base, RefExpr): - self.analyze_alias(rv) - return self.can_be_an_alias(rv.base) + return self.is_not_ready_alias(rv.base) if isinstance(rv, NameExpr): n = self.lookup(rv.name, rv) if n and isinstance(n.node, PlaceholderNode) and n.node.becomes_typeinfo: @@ -1905,7 +1909,12 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: s.is_final_def = self.unwrap_final(s) tag = self.track_incomplete_refs() s.rvalue.accept(self) - is_alias = self.can_be_an_alias(s.rvalue) + if isinstance(s.rvalue, IndexExpr) and isinstance(s.rvalue.base, RefExpr): + # Special case: analyze index expression _as a type_ to trigger + # incomplete refs for string forward references, for example + # Union['ClassA', 'ClassB'] + self.analyze_alias(s.rvalue, allow_placeholder=True) + is_alias = self.is_not_ready_alias(s.rvalue) if self.found_incomplete_ref(tag) or is_alias: # Initializer couldn't be fully analyzed. Defer the current node and give up. # Make sure that if we skip the definition of some local names, they can't be @@ -2146,8 +2155,9 @@ def analyze_simple_literal_type(self, rvalue: Expression, is_final: bool) -> Opt return None - def analyze_alias(self, rvalue: Expression) -> Tuple[Optional[Type], List[str], - Set[str], List[str]]: + def analyze_alias(self, rvalue: Expression, + allow_placeholder: bool = False) -> Tuple[Optional[Type], List[str], + Set[str], List[str]]: """Check if 'rvalue' is a valid type allowed for aliasing (e.g. not a type variable). If yes, return the corresponding type, a list of @@ -2167,6 +2177,7 @@ def analyze_alias(self, rvalue: Expression) -> Tuple[Optional[Type], List[str], self.options, self.is_typeshed_stub_file, allow_unnormalized=self.is_stub_file, + allow_placeholder=allow_placeholder, in_dynamic_func=dynamic, global_scope=global_scope) typ = None # type: Optional[Type] diff --git a/mypy/newsemanal/typeanal.py b/mypy/newsemanal/typeanal.py index 9c71a17275fac..a93a8bffbd3a4 100644 --- a/mypy/newsemanal/typeanal.py +++ b/mypy/newsemanal/typeanal.py @@ -65,6 +65,7 @@ def analyze_type_alias(node: Expression, options: Options, is_typeshed_stub: bool, allow_unnormalized: bool = False, + allow_placeholder: bool = False, in_dynamic_func: bool = False, global_scope: bool = True) -> Optional[Tuple[Type, Set[str]]]: """Analyze r.h.s. of a (potential) type alias definition. @@ -121,7 +122,8 @@ def analyze_type_alias(node: Expression, api.fail('Invalid type alias', node) return None analyzer = TypeAnalyser(api, tvar_scope, plugin, options, is_typeshed_stub, - allow_unnormalized=allow_unnormalized, defining_alias=True, allow_placeholder=True) + allow_unnormalized=allow_unnormalized, defining_alias=True, + allow_placeholder=allow_placeholder) analyzer.in_dynamic_func = in_dynamic_func analyzer.global_scope = global_scope res = type.accept(analyzer) From 31215cab9be8ef6a04535d49424a8060d7a88acf Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 15 Feb 2019 15:51:27 +0000 Subject: [PATCH 12/13] Add few more tests (skipped) --- test-data/unit/check-newsemanal.test | 36 +++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index a9a49c32efea8..0a4f8fbc521ee 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -1182,13 +1182,47 @@ class D: def test(self) -> None: reveal_type(self.x[0].y) # E: Revealed type is 'builtins.int' - class B: y: int class C: y: int [builtins fixtures/list.pyi] +[case testNewAnalyzerAliasToNotReadyTwoDeferrals-skip] +from typing import List + +x: B +B = List[C] +A = C +class C(List[A]): pass + +reveal_type(x) +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyDirectBase-skip] +from typing import List + +x: B +B = List[C] +class C(B): pass + +reveal_type(x) # E: Revealed type is 'builtins.list[__main__.C]' +reveal_type(x[0][0]) # E: Revealed type is '__main__.C' +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyMixed] +from typing import List, Union +x: A + +A = Union[B, C] + +class B(List[A]): pass +class C(List[A]): pass + +reveal_type(x) # E: Revealed type is 'Union[__main__.B, __main__.C]' +reveal_type(x[0]) # E: Revealed type is 'Union[__main__.B, __main__.C]' +[builtins fixtures/list.pyi] + [case testNewAnalyzerListComprehension] from typing import List a: List[A] From 05bb772528faf06f0d5077474525986773ccb045 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 15 Feb 2019 16:19:47 +0000 Subject: [PATCH 13/13] Add clarifying comments --- mypy/newsemanal/semanal.py | 21 ++++++++++++++------- mypy/newsemanal/typeanal.py | 3 +++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/mypy/newsemanal/semanal.py b/mypy/newsemanal/semanal.py index ad69281ed1d16..39a379430b120 100644 --- a/mypy/newsemanal/semanal.py +++ b/mypy/newsemanal/semanal.py @@ -1878,14 +1878,14 @@ def add_type_alias_deps(self, aliases_used: Iterable[str], target = self.scope.current_target() self.cur_mod_node.alias_deps[target].update(aliases_used) - def is_not_ready_alias(self, rv: Expression) -> bool: - """Does this expression look like a not ready type alias? + def is_not_ready_type_ref(self, rv: Expression) -> bool: + """Does this expression refers to a not-ready class? This includes 'Ref' and 'Ref[Arg1, Arg2, ...]', where 'Ref' refers to a PlaceholderNode with becomes_typeinfo=True. """ if isinstance(rv, IndexExpr) and isinstance(rv.base, RefExpr): - return self.is_not_ready_alias(rv.base) + return self.is_not_ready_type_ref(rv.base) if isinstance(rv, NameExpr): n = self.lookup(rv.name, rv) if n and isinstance(n.node, PlaceholderNode) and n.node.becomes_typeinfo: @@ -1905,15 +1905,22 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: if isinstance(s.rvalue, IndexExpr) and isinstance(s.rvalue.base, RefExpr): # Special case: analyze index expression _as a type_ to trigger # incomplete refs for string forward references, for example - # Union['ClassA', 'ClassB'] + # Union['ClassA', 'ClassB']. + # We throw away the results of the analysis and we only care about + # the detection of incomplete references (this doesn't change the expression + # in place). self.analyze_alias(s.rvalue, allow_placeholder=True) - is_alias = self.is_not_ready_alias(s.rvalue) - if self.found_incomplete_ref(tag) or is_alias: + top_level_not_ready = self.is_not_ready_type_ref(s.rvalue) + # NOTE: the first check is insufficient. We want to defer creation of a Var. + if self.found_incomplete_ref(tag) or top_level_not_ready: # Initializer couldn't be fully analyzed. Defer the current node and give up. # Make sure that if we skip the definition of some local names, they can't be # added later in this scope, since an earlier definition should take precedence. for expr in names_modified_by_assignment(s): - self.mark_incomplete(expr.name, expr, becomes_typeinfo=is_alias) + # NOTE: Currently for aliases like 'X = List[Y]', where 'Y' is not ready + # we proceed forward and create a Var. The latter will be replaced with + # a type alias it r.h.s. is a valid alias. + self.mark_incomplete(expr.name, expr, becomes_typeinfo=top_level_not_ready) return if self.analyze_namedtuple_assign(s): return diff --git a/mypy/newsemanal/typeanal.py b/mypy/newsemanal/typeanal.py index 5fa66b5abd89a..00b11e6b13add 100644 --- a/mypy/newsemanal/typeanal.py +++ b/mypy/newsemanal/typeanal.py @@ -409,7 +409,10 @@ def analyze_unbound_type_without_type_info(self, t: UnboundType, sym: SymbolTabl if self.allow_unbound_tvars and unbound_tvar and not self.third_pass: return t # None of the above options worked, we give up. + # NOTE: 'final_iteration' is iteration when we hit the maximum number of iterations limit. if self.api.final_iteration: + # TODO: This is problematic, since we will have to wait until the maximum number + # of iterations to report an invalid type. self.fail('Invalid type "{}"'.format(name), t) else: self.api.defer()