Optionally allow redefinition of variable with different type (#6197)

If the flag `--allow-redefinition` is used, it's now possible to redefine a 
variable with a different type. For example, this code will be okay:

```
x = 0  # Define 'x' with type 'int'
print(x)
x = 'a'  # Define another 'x' with type 'str'
print(x)
```

The variable needs to be read before redefinition is possible. This is mainly 
to allow these idioms:

```
x: List[int]
x = []  # Not a redefinition

y = None  # type: List[int]  # Strict optional disabled
y = []  # Not a redefinition
```

It's also okay to use a type annotation in any of the assignments. This is
fine:

```
x: int = 0
print(x)
x: str = 'a'
print(x)
```

Redefinition is only allowed to happen in the same block and the nesting level
as the original definition. This lets us keep the implementation relatively simple,
and there will hopefully be less of a readability impact, since it's less likely that
users accidentally redefine a variable or use redefinition in confusing ways. We 
may want to lift the restriction in the future, however. Function arguments can
be redefined in the body but only in non-nested blocks.

The implementation does a simple static analysis to rename variables to make
them distinct when we detect redefinition. This happens before semantic 
analysis pass 1.

The new behavior is not fully backward compatible with the old behavior. For
example, here we lose type context in the second assignment:

```
x: List[int] = []
f(x)
x = []  # Need type annotation since this creates an independent variable
```

Internally we use a sequence of single quotes as a variable suffix
to represent renamed variants. The implementation provides an `unmangle()`
function to get the original name from a potentially mangled name.

It's still impossible to redefine final names, classes and certain other things.
The main use case is simple variables, either in single or multiple assignments.

I did some benchmarking (not with the latest version though) and the performance 
impact of the new pass was minor (under 2% IIRC).

Things to do in additional PRs:
* Add documentation.
* Make sure we unmangle names everywhere.
* Consider allowing redefinition to happen sometimes in another block. 
* Consider propagating type context from original definition.

Author: JukkaL

mypy/checkmember.py

@@ -9,7 +9,8 @@
 )
 from mypy.nodes import (
     TypeInfo, FuncBase, Var, FuncDef, SymbolNode, Context, MypyFile, TypeVarExpr,
-    ARG_POS, ARG_STAR, ARG_STAR2, Decorator, OverloadedFuncDef, TypeAlias, TempNode
+    ARG_POS, ARG_STAR, ARG_STAR2, Decorator, OverloadedFuncDef, TypeAlias, TempNode,
+    is_final_node
 )
 from mypy.messages import MessageBuilder
 from mypy.maptype import map_instance_to_supertype
@@ -895,8 +896,3 @@ def erase_to_bound(t: Type) -> Type:
         if isinstance(t.item, TypeVarType):
             return TypeType.make_normalized(t.item.upper_bound)
     return t
-
-
-def is_final_node(node: Optional[SymbolNode]) -> bool:
-    """Check whether `node` corresponds to a final attribute."""
-    return isinstance(node, (Var, FuncDef, OverloadedFuncDef, Decorator)) and node.is_final

mypy/main.py

@@ -523,6 +523,10 @@ def add_invertible_flag(flag: str,
                         help="Suppress toplevel errors caused by missing annotations",
                         group=strictness_group)
 
+    add_invertible_flag('--allow-redefinition', default=False, strict_flag=False,
+                        help="Allow unconditional variable redefinition with a new type",
+                        group=strictness_group)
+
     incremental_group = parser.add_argument_group(
         title='Incremental mode',
         description="Adjust how mypy incrementally type checks and caches modules. "

mypy/messages.py

@@ -29,6 +29,7 @@
     ReturnStmt, NameExpr, Var, CONTRAVARIANT, COVARIANT, SymbolNode,
     CallExpr
 )
+from mypy.util import unmangle
 from mypy import message_registry
 
 MYPY = False
@@ -952,7 +953,7 @@ def cant_assign_to_final(self, name: str, attr_assign: bool, ctx: Context) -> No
         Pass `attr_assign=True` if the assignment assigns to an attribute.
         """
         kind = "attribute" if attr_assign else "name"
-        self.fail('Cannot assign to final {} "{}"'.format(kind, name), ctx)
+        self.fail('Cannot assign to final {} "{}"'.format(kind, unmangle(name)), ctx)
 
     def protocol_members_cant_be_final(self, ctx: Context) -> None:
         self.fail("Protocol member cannot be final", ctx)
@@ -1064,7 +1065,7 @@ def unimported_type_becomes_any(self, prefix: str, typ: Type, ctx: Context) -> N
                   ctx)
 
     def need_annotation_for_var(self, node: SymbolNode, context: Context) -> None:
-        self.fail("Need type annotation for '{}'".format(node.name()), context)
+        self.fail("Need type annotation for '{}'".format(unmangle(node.name())), context)
 
     def explicit_any(self, ctx: Context) -> None:
         self.fail('Explicit "Any" is not allowed', ctx)

mypy/nodes.py

@@ -924,15 +924,18 @@ def accept(self, visitor: StatementVisitor[T]) -> T:
 
 
 class AssignmentStmt(Statement):
-    """Assignment statement
+    """Assignment statement.
+
     The same node class is used for single assignment, multiple assignment
     (e.g. x, y = z) and chained assignment (e.g. x = y = z), assignments
-    that define new names, and assignments with explicit types (# type).
+    that define new names, and assignments with explicit types ("# type: t"
+    or "x: t [= ...]").
 
-    An lvalue can be NameExpr, TupleExpr, ListExpr, MemberExpr, IndexExpr.
+    An lvalue can be NameExpr, TupleExpr, ListExpr, MemberExpr, or IndexExpr.
     """
 
     lvalues = None  # type: List[Lvalue]
+    # This is a TempNode if and only if no rvalue (x: t).
     rvalue = None  # type: Expression
     # Declared type in a comment, may be None.
     type = None  # type: Optional[mypy.types.Type]
@@ -2968,3 +2971,8 @@ def is_class_var(expr: NameExpr) -> bool:
     if isinstance(expr.node, Var):
         return expr.node.is_classvar
     return False
+
+
+def is_final_node(node: Optional[SymbolNode]) -> bool:
+    """Check whether `node` corresponds to a final attribute."""
+    return isinstance(node, (Var, FuncDef, OverloadedFuncDef, Decorator)) and node.is_final

mypy/options.py

@@ -21,6 +21,7 @@ class BuildType:
 PER_MODULE_OPTIONS = {
     # Please keep this list sorted
     "allow_untyped_globals",
+    "allow_redefinition",
     "always_false",
     "always_true",
     "check_untyped_defs",
@@ -149,6 +150,10 @@ def __init__(self) -> None:
         # Suppress toplevel errors caused by missing annotations
         self.allow_untyped_globals = False
 
+        # Allow variable to be redefined with an arbitrary type in the same block
+        # and the same nesting level as the initialization
+        self.allow_redefinition = False
+
         # Variable names considered True
         self.always_true = []  # type: List[str]
 

mypy/plugins/attrs.py

@@ -19,6 +19,7 @@
     Overloaded, UnionType, FunctionLike
 )
 from mypy.typevars import fill_typevars
+from mypy.util import unmangle
 from mypy.server.trigger import make_wildcard_trigger
 
 MYPY = False
@@ -376,9 +377,10 @@ def _attribute_from_auto_attrib(ctx: 'mypy.plugin.ClassDefContext',
                                 rvalue: Expression,
                                 stmt: AssignmentStmt) -> Attribute:
     """Return an Attribute for a new type assignment."""
+    name = unmangle(lhs.name)
     # `x: int` (without equal sign) assigns rvalue to TempNode(AnyType())
     has_rhs = not isinstance(rvalue, TempNode)
-    return Attribute(lhs.name, ctx.cls.info, has_rhs, True, kw_only, Converter(), stmt)
+    return Attribute(name, ctx.cls.info, has_rhs, True, kw_only, Converter(), stmt)
 
 
 def _attribute_from_attrib_maker(ctx: 'mypy.plugin.ClassDefContext',
@@ -443,7 +445,8 @@ def _attribute_from_attrib_maker(ctx: 'mypy.plugin.ClassDefContext',
         converter = convert
     converter_info = _parse_converter(ctx, converter)
 
-    return Attribute(lhs.name, ctx.cls.info, attr_has_default, init, kw_only, converter_info, stmt)
+    name = unmangle(lhs.name)
+    return Attribute(name, ctx.cls.info, attr_has_default, init, kw_only, converter_info, stmt)
 
 
 def _parse_converter(ctx: 'mypy.plugin.ClassDefContext',