From 230911c9492770d3def2fb33baaf62304b4ddceb Mon Sep 17 00:00:00 2001 From: David Fisher Date: Fri, 13 Jan 2017 14:52:28 -0800 Subject: [PATCH 1/4] Fast parser: support @no_type_check, give better ellipses error, fix up more tests --- mypy/fastparse.py | 35 ++++++++++++++++++++++++----- mypy/fastparse2.py | 21 ++++++++++++++--- mypy/semanal.py | 3 ++- mypy/test/testcheck.py | 6 ++--- test-data/unit/check-columns.test | 8 +++---- test-data/unit/check-functions.test | 4 ++-- test-data/unit/check-python2.test | 2 +- test-data/unit/check-tuples.test | 2 +- 8 files changed, 61 insertions(+), 20 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index baaac3c069fb0..c0dedaa09aff4 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -108,6 +108,15 @@ def find(f: Callable[[V], bool], seq: Sequence[V]) -> V: return None +def is_no_type_check_decorator(expr: ast35.expr) -> bool: + if isinstance(expr, ast35.Name): + return expr.id == 'no_type_check' + elif isinstance(expr, ast35.Attribute): + if isinstance(expr.value, ast35.Name): + return expr.value.id == 'typing' and expr.attr == 'no_type_check' + return False + + class ASTConverter(ast35.NodeTransformer): def __init__(self, pyversion: Tuple[int, int], @@ -268,7 +277,10 @@ def visit_AsyncFunctionDef(self, n: ast35.AsyncFunctionDef) -> Union[FuncDef, De def do_func_def(self, n: Union[ast35.FunctionDef, ast35.AsyncFunctionDef], is_coroutine: bool = False) -> Union[FuncDef, Decorator]: """Helper shared between visit_FunctionDef and visit_AsyncFunctionDef.""" - args = self.transform_args(n.args, n.lineno) + no_type_check = bool(n.decorator_list and + any(is_no_type_check_decorator(d) for d in n.decorator_list)) + + args = self.transform_args(n.args, n.lineno, no_type_check=no_type_check) arg_kinds = [arg.kind for arg in args] arg_names = [arg.variable.name() for arg in args] # type: List[Optional[str]] @@ -276,7 +288,10 @@ def do_func_def(self, n: Union[ast35.FunctionDef, ast35.AsyncFunctionDef], if special_function_elide_names(n.name): arg_names = [None] * len(arg_names) arg_types = None # type: List[Type] - if n.type_comment is not None: + if no_type_check: + arg_types = [None for _ in args] + return_type = None + elif n.type_comment is not None: try: func_type_ast = ast35.parse(n.type_comment, '', 'func_type') assert isinstance(func_type_ast, ast35.FunctionType) @@ -312,7 +327,10 @@ def do_func_def(self, n: Union[ast35.FunctionDef, ast35.AsyncFunctionDef], func_type = None if any(arg_types) or return_type: - if len(arg_types) > len(arg_kinds): + if len(arg_types) != 1 and any(isinstance(t, EllipsisType) for t in arg_types): + self.fail("Ellipses cannot accompany other argument types " + "in function type signature.", n.lineno, 0) + elif len(arg_types) > len(arg_kinds): self.fail('Type signature has too many arguments', n.lineno, 0) elif len(arg_types) < len(arg_kinds): self.fail('Type signature has too few arguments', n.lineno, 0) @@ -356,9 +374,16 @@ def set_type_optional(self, type: Type, initializer: Expression) -> None: if isinstance(type, UnboundType): type.optional = optional - def transform_args(self, args: ast35.arguments, line: int) -> List[Argument]: + def transform_args(self, + args: ast35.arguments, + line: int, + no_type_check: bool = False, + ) -> List[Argument]: def make_argument(arg: ast35.arg, default: Optional[ast35.expr], kind: int) -> Argument: - arg_type = TypeConverter(self.errors, line=line).visit(arg.annotation) + if no_type_check: + arg_type = None + else: + arg_type = TypeConverter(self.errors, line=line).visit(arg.annotation) return Argument(Var(arg.arg), arg_type, self.visit(default), kind) new_args = [] diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 5acde8f18758a..310a7a116905e 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -34,7 +34,7 @@ ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2 ) from mypy.types import ( - Type, CallableType, AnyType, UnboundType, + Type, CallableType, AnyType, UnboundType, EllipsisType ) from mypy import defaults from mypy import experiments @@ -114,6 +114,15 @@ def find(f: Callable[[V], bool], seq: Sequence[V]) -> V: return None +def is_no_type_check_decorator(expr: ast27.expr) -> bool: + if isinstance(expr, ast27.Name): + return expr.id == 'no_type_check' + elif isinstance(expr, ast27.Attribute): + if isinstance(expr.value, ast27.Name): + return expr.value.id == 'typing' and expr.attr == 'no_type_check' + return False + + class ASTConverter(ast27.NodeTransformer): def __init__(self, pyversion: Tuple[int, int], @@ -274,7 +283,10 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: arg_names = [None] * len(arg_names) arg_types = None # type: List[Type] - if n.type_comment is not None and len(n.type_comment) > 0: + if (n.decorator_list and any(is_no_type_check_decorator(d) for d in n.decorator_list)): + arg_types = [None for _ in args] + return_type = None + elif n.type_comment is not None and len(n.type_comment) > 0: try: func_type_ast = ast35.parse(n.type_comment, '', 'func_type') assert isinstance(func_type_ast, ast35.FunctionType) @@ -307,7 +319,10 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: func_type = None if any(arg_types) or return_type: - if len(arg_types) > len(arg_kinds): + if len(arg_types) != 1 and any(isinstance(t, EllipsisType) for t in arg_types): + self.fail("Ellipses cannot accompany other argument types " + "in function type signature.", n.lineno, 0) + elif len(arg_types) > len(arg_kinds): self.fail('Type signature has too many arguments', n.lineno, 0) elif len(arg_types) < len(arg_kinds): self.fail('Type signature has too few arguments', n.lineno, 0) diff --git a/mypy/semanal.py b/mypy/semanal.py index aa78a38026427..b3eccdbb7655d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1404,10 +1404,11 @@ def store_declared_types(self, lvalue: Lvalue, typ: Type) -> None: self.fail('Tuple type expected for multiple variables', lvalue) elif isinstance(lvalue, StarExpr): + # Historical behavior for the old parser if isinstance(typ, StarType): self.store_declared_types(lvalue.expr, typ.type) else: - self.fail('Star type expected for starred expression', lvalue) + self.store_declared_types(lvalue.expr, typ) else: # This has been flagged elsewhere as an error, so just ignore here. pass diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 027331dd30720..e12004ba91426 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -27,11 +27,8 @@ # List of files that contain test case descriptions. files = [ - 'check-columns.test', 'check-expressions.test', - 'check-functions.test', 'check-generic-subtyping.test', - 'check-tuples.test', 'check-varargs.test', ] fast_parser_files = [ @@ -73,6 +70,9 @@ 'check-class-namedtuple.test', 'check-selftype.test', 'check-python2.test', + 'check-columns.test', + 'check-functions.test', + 'check-tuples.test', ] files.extend(fast_parser_files) diff --git a/test-data/unit/check-columns.test b/test-data/unit/check-columns.test index ae8bab87c74b9..7e21ca09a7fc9 100644 --- a/test-data/unit/check-columns.test +++ b/test-data/unit/check-columns.test @@ -2,7 +2,7 @@ # flags: --show-column-numbers 1 + [out] -main:2:3: error: Parse error before end of line +main:2:4: error: invalid syntax [case testColumnsNestedFunctions] @@ -40,9 +40,9 @@ class A: pass A().f() A().f(1) -A().f('') # E:5: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" -A().f(1, 1) # E:5: Argument 2 to "f" of "A" has incompatible type "int"; expected "str" -A().f(1, 'hello', 'hi') # E:5: Too many arguments for "f" of "A" +A().f('') # E:0: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" +A().f(1, 1) # E:0: Argument 2 to "f" of "A" has incompatible type "int"; expected "str" +A().f(1, 'hello', 'hi') # E:0: Too many arguments for "f" of "A" [case testColumnsMultipleStatementsPerLine] # flags: --show-column-numbers diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 31a61688eeeeb..5fb89326ce8a7 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1475,13 +1475,13 @@ class A: def f(x, y, z): # type: (..., int) -> None pass [out] -main:1: error: Parse error before ): Ellipses cannot accompany other argument types in function type signature. +main:1: error: Ellipses cannot accompany other argument types in function type signature. [case testEllipsisWithSomethingBeforeItFails] def f(x, y, z): # type: (int, ...) -> None pass [out] -main:1: error: Parse error before ): Ellipses cannot accompany other argument types in function type signature. +main:1: error: Ellipses cannot accompany other argument types in function type signature. [case testRejectCovariantArgument] from typing import TypeVar, Generic diff --git a/test-data/unit/check-python2.test b/test-data/unit/check-python2.test index e5ce179b72fea..7a9446b4f495b 100644 --- a/test-data/unit/check-python2.test +++ b/test-data/unit/check-python2.test @@ -183,7 +183,7 @@ def f(x, y, z): # type: (...) -> None def f(x, y, z): # type: (..., int) -> None pass [out] -main:1: error: Type signature has too few arguments +main:1: error: Ellipses cannot accompany other argument types in function type signature. [case testLambdaTupleArgInPython2] f = lambda (x, y): x + y diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index 50d6a2d6409d4..13c8ba4558ea0 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -387,7 +387,7 @@ aa, bb, *cc = t # E: Need type annotation for variable [case testAssignmentToStarAnnotation] from typing import List li, lo = None, None # type: List[int], List[object] -a, b, *c = 1, 2 # type: int, int, *List[int] +a, b, *c = 1, 2 # type: int, int, List[int] c = lo # E: Incompatible types in assignment (expression has type List[object], variable has type List[int]) c = li [builtins fixtures/list.pyi] From a33f618203ecfc659d69c82e6c91a9c7a1fdaa7d Mon Sep 17 00:00:00 2001 From: David Fisher Date: Thu, 19 Jan 2017 14:56:11 -0800 Subject: [PATCH 2/4] Remove outdated semanal tests --- test-data/unit/semanal-errors.test | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 06fd3d0dd2261..3d3a777a8ec72 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -149,16 +149,6 @@ z = 0 # type: x main:4: error: Invalid type "__main__.f" main:5: error: Invalid type "__main__.x" -[case testTwoStarsInType] -import typing -x, x2 = 1 # type: *object, *object -y, y2 = 1 # type: object, (*object, *object) -z, z2 = 1 # type: *object, (object, *object) -[out] -main:2: error: At most one star type allowed in a tuple -main:3: error: At most one star type allowed in a tuple -main:4: error: Star type only allowed for starred expressions - [case testGlobalVarRedefinition] import typing class A: pass @@ -419,16 +409,6 @@ main:4: error: Invalid assignment target main:5: error: Invalid assignment target main:6: error: Invalid assignment target -[case testInvalidStarType] -a = 1 # type: *int -[out] -main:1: error: Star type only allowed for starred expressions - -[case testInvalidStarType] -*a, b = 1 # type: int, int -[out] -main:1: error: Star type expected for starred expression - [case testTwoStarExpressions] a, *b, *c = 1 *a, (*b, c) = 1 From 3e1f3e7209611de54e7fe54d05fc5cb8306c9c93 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Mon, 23 Jan 2017 12:35:13 -0800 Subject: [PATCH 3/4] fix review nit --- mypy/fastparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index c0dedaa09aff4..97f7e53d94e21 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -289,7 +289,7 @@ def do_func_def(self, n: Union[ast35.FunctionDef, ast35.AsyncFunctionDef], arg_names = [None] * len(arg_names) arg_types = None # type: List[Type] if no_type_check: - arg_types = [None for _ in args] + arg_types = [None] * len(args) return_type = None elif n.type_comment is not None: try: From ec411dca423ad85d5630073d90b49a10dabf27f3 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Mon, 23 Jan 2017 14:13:04 -0800 Subject: [PATCH 4/4] Fix other instances of unidiomatic pattern --- mypy/fastparse.py | 4 ++-- mypy/fastparse2.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 97f7e53d94e21..497485a7d6590 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -313,7 +313,7 @@ def do_func_def(self, n: Union[ast35.FunctionDef, ast35.AsyncFunctionDef], arg_types.insert(0, AnyType()) except SyntaxError: self.fail(TYPE_COMMENT_SYNTAX_ERROR, n.lineno, n.col_offset) - arg_types = [AnyType() for _ in args] + arg_types = [AnyType()] * len(args) return_type = AnyType() else: arg_types = [a.type_annotation for a in args] @@ -777,7 +777,7 @@ def is_star2arg(k: ast35.keyword) -> bool: return CallExpr(self.visit(n.func), arg_types, arg_kinds, - cast("List[str]", [None for _ in n.args]) + [k.arg for k in n.keywords]) + cast("List[str]", [None] * len(n.args)) + [k.arg for k in n.keywords]) # Num(object n) -- a number as a PyObject. @with_line diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 310a7a116905e..895fd7a183e1f 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -284,7 +284,7 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: arg_types = None # type: List[Type] if (n.decorator_list and any(is_no_type_check_decorator(d) for d in n.decorator_list)): - arg_types = [None for _ in args] + arg_types = [None] * len(args) return_type = None elif n.type_comment is not None and len(n.type_comment) > 0: try: @@ -305,7 +305,7 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: arg_types.insert(0, AnyType()) except SyntaxError: self.fail(TYPE_COMMENT_SYNTAX_ERROR, n.lineno, n.col_offset) - arg_types = [AnyType() for _ in args] + arg_types = [AnyType()] * len(args) return_type = AnyType() else: arg_types = [a.type_annotation for a in args]