From 1954798a6e982e00c4dc990c96a33db01bb1a99c Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 17:47:31 +0200 Subject: [PATCH 01/20] implement ast-transformer for return location --- python/deps/untypy/untypy/__init__.py | 15 +++++++-------- python/deps/untypy/untypy/error.py | 12 ++++++++++++ .../untypy/untypy/patching/ast_transformer.py | 2 +- python/deps/untypy/untypy/util/__init__.py | 8 +++++++- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/python/deps/untypy/untypy/__init__.py b/python/deps/untypy/untypy/__init__.py index 9d24b60d..e96f0051 100644 --- a/python/deps/untypy/untypy/__init__.py +++ b/python/deps/untypy/untypy/__init__.py @@ -9,9 +9,10 @@ UntypyAstImportTransformer from .patching.import_hook import install_import_hook from .util.condition import FunctionCondition +from .util.return_traces import ReturnTracesTransformer, before_return, GlobalReturnTraceManager GlobalConfig = DefaultConfig - +_before_return = before_return def just_install_hook(prefixes=[]): def predicate(module_name): @@ -22,18 +23,15 @@ def predicate(module_name): return True return False - install_import_hook(predicate, lambda path: UntypyAstTransformer()) - + install_import_hook(predicate, lambda path: UntypyAstTransformer()) # TODO: ReturnTracesTransformer FIX ME!!! -def just_transform(source, modname, symbol='exec'): - tree = compile(source, modname, symbol, flags=ast.PyCF_ONLY_AST, dont_inherit=True, optimize=-1) - transform_tree(tree) - return tree -def transform_tree(tree): +def transform_tree(tree, file): UntypyAstTransformer().visit(tree) + ReturnTracesTransformer(file).visit(tree) ast.fix_missing_locations(tree) + def enable(*, recursive: bool = True, root: Union[ModuleType, str, None] = None, prefixes: list[str] = []) -> None: global GlobalConfig caller = _find_calling_module() @@ -104,6 +102,7 @@ def _exec_module_patched(mod: ModuleType, exit_after: bool, transformer: ast.Nod "\tuntypy.enable()") transformer.visit(tree) + ReturnTracesTransformer(lambda r: GlobalReturnTraceManager.next_id(r, mod.__file__)).visit(tree) ast.fix_missing_locations(tree) patched_mod = compile(tree, mod.__file__, 'exec', dont_inherit=True, optimize=-1) stack = list(map(lambda s: s.frame, inspect.stack())) diff --git a/python/deps/untypy/untypy/error.py b/python/deps/untypy/untypy/error.py index 05a264f2..ea98327e 100644 --- a/python/deps/untypy/untypy/error.py +++ b/python/deps/untypy/untypy/error.py @@ -78,6 +78,18 @@ def from_stack(stack) -> Location: source_line=None ) + def mark(self, reti_loc): + file, line = reti_loc + lines = self.source_line.splitlines() + if self.file == file and line in range(self.line_no, self.line_no + len(lines)): + return Location( + file=file, + line_no=line, + source_line=lines[line - self.line_no] + ) + else: + return self + class Frame: type_declared: str diff --git a/python/deps/untypy/untypy/patching/ast_transformer.py b/python/deps/untypy/untypy/patching/ast_transformer.py index 6f953f37..f48ee49b 100644 --- a/python/deps/untypy/untypy/patching/ast_transformer.py +++ b/python/deps/untypy/untypy/patching/ast_transformer.py @@ -1,5 +1,5 @@ import ast -from typing import Callable, List, Optional +from typing import Callable, List, Optional, Any class UntypyAstTransformer(ast.NodeTransformer): diff --git a/python/deps/untypy/untypy/util/__init__.py b/python/deps/untypy/untypy/util/__init__.py index 52bc4234..9c76dc31 100644 --- a/python/deps/untypy/untypy/util/__init__.py +++ b/python/deps/untypy/untypy/util/__init__.py @@ -5,6 +5,7 @@ from untypy.display import IndicatorStr from untypy.error import UntypyTypeError, Frame, Location from untypy.interfaces import ExecutionContext, TypeChecker, WrappedFunction +from untypy.util.return_traces import get_last_return class ReplaceTypeExecutionContext(ExecutionContext): @@ -104,6 +105,7 @@ class ReturnExecutionContext(ExecutionContext): fn: WrappedFunction def __init__(self, fn: WrappedFunction): + self.reti_loc = get_last_return() self.fn = fn def wrap(self, err: UntypyTypeError) -> UntypyTypeError: @@ -123,11 +125,15 @@ def wrap(self, err: UntypyTypeError) -> UntypyTypeError: return_id = IndicatorStr("???") declared = WrappedFunction.find_location(self.fn) + responsable = declared + if responsable is not None: + responsable = responsable.mark(self.reti_loc) + return err.with_frame(Frame( return_id.ty, return_id.indicator, declared=declared, - responsable=declared, + responsable=responsable, )) From cc6d50b4fe27b527c12315413af7904fc2090461 Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 17:56:12 +0200 Subject: [PATCH 02/20] implement ast-transformer for return location II --- .../deps/untypy/untypy/util/return_traces.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 python/deps/untypy/untypy/util/return_traces.py diff --git a/python/deps/untypy/untypy/util/return_traces.py b/python/deps/untypy/untypy/util/return_traces.py new file mode 100644 index 00000000..6b78d5af --- /dev/null +++ b/python/deps/untypy/untypy/util/return_traces.py @@ -0,0 +1,66 @@ +import ast +from typing import * + + +class ReturnTraceManager: + def __init__(self): + self.lst = [] + + def next_id(self, reti : ast.Return, file: str) -> int: + i = len(self.lst) + self.lst.append((file, reti.lineno)) + return i + + def get(self, idx : int) -> (str, int): + return self.lst[idx] + +GlobalReturnTraceManager = ReturnTraceManager() +reti_loc: int = -1 + +def before_return(idx : int): + global reti_loc + reti_loc = idx + +def get_last_return() -> (str, int): + global reti_loc + if reti_loc < 0: + return ("", 0) # this will never match any real location + return GlobalReturnTraceManager.get(reti_loc) + + +class ReturnTracesTransformer(ast.NodeTransformer): + + def __init__(self, registry: Callable[[ast.Return], int]): + self.registry = registry + + def generic_visit(self, node) -> Any: + # See https://docs.python.org/3/library/ast.html + stmt_types = ["body", "orelse", "finalbody"] + + for stmt_type in stmt_types: + if not hasattr(node, stmt_type): + continue + statements : list = getattr(node, stmt_type) + + if not isinstance(statements, Iterable): + # skip lambda expressions + continue + + inserts = [] + for i,s in enumerate(statements): + if type(s) is ast.Return: + inserts.append((i, s)) + + # start at end so the early indexes still fit + inserts.reverse() + + for index, reti in inserts: + n = ast.Expr(value=ast.Call( + func=ast.Attribute(value=ast.Name(id='untypy', ctx=ast.Load()), attr='_before_return', ctx=ast.Load()), + args=[ast.Constant(value=self.registry(reti))], + keywords=[]) + ) + statements.insert(index, n) + + super(ast.NodeTransformer, self).generic_visit(node) + From 24e9dd6b045da458ab03d36f7c60a1e159eaa7aa Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 18:20:46 +0200 Subject: [PATCH 03/20] refactor Location --- python/deps/untypy/untypy/error.py | 60 ++++++++++++++++--------- python/deps/untypy/untypy/interfaces.py | 2 +- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/python/deps/untypy/untypy/error.py b/python/deps/untypy/untypy/error.py index ea98327e..e8703ad7 100644 --- a/python/deps/untypy/untypy/error.py +++ b/python/deps/untypy/untypy/error.py @@ -2,31 +2,51 @@ import inspect from enum import Enum +from os.path import relpath from typing import Any, Optional, Tuple, Iterable class Location: file: str line_no: int - source_line: str + line_span : int + source_lines: Optional[str] - def __init__(self, file: str, line_no: int, source_line: str): + def __init__(self, file: str, line_no: int, line_span : int): self.file = file self.line_no = line_no - self.source_line = source_line + self.line_span = line_span + self.source_line = None + + def _source(self) -> Optional[str]: + if self.source_line is None: + try: + with open(self.file, "r") as f: + self.source_lines = f.read() + except OSError: + pass + + return self.source_lines def __str__(self): - buf = f"{self.file}:{self.line_no}" - if self.source_line: - for i, line in enumerate(self.source_line.splitlines()): - if i < 5: - buf += f"\n{'{:3}'.format(self.line_no + i)} | {line}" - if i >= 5: - buf += "\n | ..." + buf = f"{relpath(self.file)}:{self.line_no}" + source = self._source() + if source is None: + buf += f"\n{'{:3}'.format(self.line_no)} | " + return buf + + start = max(self.line_no - 2, 1) + end = start + 5 + for i, line in enumerate(source.splitlines()): + if (i + 1) == self.line_no: + buf += f"\n{'{:3}'.format(i + 1)} > {line}" + elif (i + 1) in range(start, end): + buf += f"\n{'{:3}'.format(i + 1)} | {line}" + return buf def __repr__(self): - return f"Location(file={self.file.__repr__()}, line_no={self.line_no.__repr__()}, source_line={repr(self.source_line)})" + return f"Location(file={self.file.__repr__()}, line_no={self.line_no.__repr__()}, line_span={self.line_span})" def __eq__(self, other): if not isinstance(other, Location): @@ -39,13 +59,13 @@ def from_code(obj) -> Location: return Location( file=inspect.getfile(obj), line_no=inspect.getsourcelines(obj)[1], - source_line="".join(inspect.getsourcelines(obj)[0]), + line_span=len(inspect.getsourcelines(obj)[0]), ) except Exception: return Location( file=inspect.getfile(obj), line_no=1, - source_line=repr(obj) + line_span=1, ) @staticmethod @@ -55,37 +75,35 @@ def from_stack(stack) -> Location: return Location( file=stack.filename, line_no=stack.lineno, - source_line=stack.code_context[0] + line_span=1 ) except Exception: return Location( file=stack.filename, line_no=stack.lineno, - source_line=None + line_span=1 ) else: # assume sys._getframe(...) try: - source_line = inspect.findsource(stack.f_code)[0][stack.f_lineno - 1] return Location( file=stack.f_code.co_filename, line_no=stack.f_lineno, - source_line=source_line + line_span=1 ) except Exception: return Location( file=stack.f_code.co_filename, line_no=stack.f_lineno, - source_line=None + line_span=1 ) def mark(self, reti_loc): file, line = reti_loc - lines = self.source_line.splitlines() - if self.file == file and line in range(self.line_no, self.line_no + len(lines)): + if self.file == file and line in range(self.line_no, self.line_no + self.line_span): return Location( file=file, line_no=line, - source_line=lines[line - self.line_no] + line_span=1 ) else: return self diff --git a/python/deps/untypy/untypy/interfaces.py b/python/deps/untypy/untypy/interfaces.py index 9207ad7a..6e904a2c 100644 --- a/python/deps/untypy/untypy/interfaces.py +++ b/python/deps/untypy/untypy/interfaces.py @@ -96,7 +96,7 @@ def find_location(fn) -> Optional[Location]: return Location( file=inspect.getfile(fn), line_no=inspect.getsourcelines(fn)[1], - source_line="".join(inspect.getsourcelines(fn)[0]), + line_span=len(inspect.getsourcelines(fn)[0]), ) except: # Failes on builtins return None From c917fcb577e6d88c6d35a74500383da2b67be80f Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 18:33:50 +0200 Subject: [PATCH 04/20] refactor Location II --- .../untypy/test/impl/test_bound_generic.py | 4 ++-- python/deps/untypy/test/util.py | 4 ++-- python/deps/untypy/untypy/error.py | 23 +++++++++++++++---- python/deps/untypy/untypy/impl/generator.py | 2 +- python/deps/untypy/untypy/impl/iterator.py | 2 +- python/deps/untypy/untypy/impl/protocol.py | 2 +- .../deps/untypy/untypy/patching/__init__.py | 4 ++-- 7 files changed, 28 insertions(+), 13 deletions(-) diff --git a/python/deps/untypy/test/impl/test_bound_generic.py b/python/deps/untypy/test/impl/test_bound_generic.py index 1747b455..a018b4b3 100644 --- a/python/deps/untypy/test/impl/test_bound_generic.py +++ b/python/deps/untypy/test/impl/test_bound_generic.py @@ -35,7 +35,7 @@ def test_bound_generic_caller_error(self): with self.assertRaises(UntypyTypeError) as cm: instance.insert("this should be an int") - self.assertTrue("instance.insert" in cm.exception.last_responsable().source_line) + self.assertTrue("instance.insert" in cm.exception.last_responsable().source_lines_span()) def test_bound_generic_protocol_style_ok(self): instance = Aint() @@ -99,4 +99,4 @@ def test_bound_generic_return_error(self): with self.assertRaises(UntypyTypeError) as cm: instance.some_string() - self.assertTrue("def some_string(self) -> T:" in cm.exception.last_responsable().source_line) + self.assertTrue("def some_string(self) -> T:" in cm.exception.last_responsable().source_lines_span()) diff --git a/python/deps/untypy/test/util.py b/python/deps/untypy/test/util.py index 9993187c..6e20f8f2 100644 --- a/python/deps/untypy/test/util.py +++ b/python/deps/untypy/test/util.py @@ -16,7 +16,7 @@ def wrap(self, err: UntypyTypeError) -> UntypyTypeError: responsable=Location( file="dummy", line_no=0, - source_line="dummy" + line_span=1 ) )) @@ -27,7 +27,7 @@ def __init__(self, typevars: dict[TypeVar, Any] = dict()): super().__init__(typevars.copy(), Location( file="dummy", line_no=0, - source_line="dummy" + line_span=1 ), checkedpkgprefixes=["test"]) diff --git a/python/deps/untypy/untypy/error.py b/python/deps/untypy/untypy/error.py index e8703ad7..74d0da62 100644 --- a/python/deps/untypy/untypy/error.py +++ b/python/deps/untypy/untypy/error.py @@ -16,10 +16,10 @@ def __init__(self, file: str, line_no: int, line_span : int): self.file = file self.line_no = line_no self.line_span = line_span - self.source_line = None + self.source_lines = None - def _source(self) -> Optional[str]: - if self.source_line is None: + def source(self) -> Optional[str]: + if self.source_lines is None: try: with open(self.file, "r") as f: self.source_lines = f.read() @@ -28,9 +28,24 @@ def _source(self) -> Optional[str]: return self.source_lines + def source_lines_span(self) -> Optional[str]: + # This is still used for unit testing + + source = self.source() + if source is None: + return None + + buf = "" + + for i, line in enumerate(source.splitlines()): + if (i + 1) in range(self.line_no, self.line_no + self.line_span): + buf += f"\n{line}" + + return buf + def __str__(self): buf = f"{relpath(self.file)}:{self.line_no}" - source = self._source() + source = self.source() if source is None: buf += f"\n{'{:3}'.format(self.line_no)} | " return buf diff --git a/python/deps/untypy/untypy/impl/generator.py b/python/deps/untypy/untypy/impl/generator.py index 82812a21..6551bdb3 100644 --- a/python/deps/untypy/untypy/impl/generator.py +++ b/python/deps/untypy/untypy/impl/generator.py @@ -106,7 +106,7 @@ def responsable(self) -> Optional[Location]: return Location( file=inspect.getfile(self.generator.gi_frame), line_no=inspect.getsourcelines(self.generator.gi_frame)[1], - source_line="\n".join(inspect.getsourcelines(self.generator.gi_frame)[0]), + line_span=len(inspect.getsourcelines(self.generator.gi_frame)[0]), ) except OSError: # this call does not work all the time pass diff --git a/python/deps/untypy/untypy/impl/iterator.py b/python/deps/untypy/untypy/impl/iterator.py index cc2c1dd4..89a34cf4 100644 --- a/python/deps/untypy/untypy/impl/iterator.py +++ b/python/deps/untypy/untypy/impl/iterator.py @@ -71,7 +71,7 @@ def responsable(self) -> Optional[Location]: return Location( file=inspect.getfile(self.iter.gi_frame), line_no=inspect.getsourcelines(self.iter.gi_frame)[1], - source_line="\n".join(inspect.getsourcelines(self.iter.gi_frame)[0]), + line_span=len(inspect.getsourcelines(self.iter.gi_frame)[0]), ) except OSError: # this call does not work all the time pass diff --git a/python/deps/untypy/untypy/impl/protocol.py b/python/deps/untypy/untypy/impl/protocol.py index 19aa967f..c9290541 100644 --- a/python/deps/untypy/untypy/impl/protocol.py +++ b/python/deps/untypy/untypy/impl/protocol.py @@ -38,7 +38,7 @@ def _find_bound_typevars(clas: type) -> (type, Dict[TypeVar, Any]): [Location( file=inspect.getfile(clas), line_no=inspect.getsourcelines(clas)[1], - source_line="".join(inspect.getsourcelines(clas)[0]))]) + line_span=len(inspect.getsourcelines(clas)[0]))]) return (clas.__origin__, dict(zip(keys, values))) diff --git a/python/deps/untypy/untypy/patching/__init__.py b/python/deps/untypy/untypy/patching/__init__.py index 2ee22cbd..a39576e0 100644 --- a/python/deps/untypy/untypy/patching/__init__.py +++ b/python/deps/untypy/untypy/patching/__init__.py @@ -28,7 +28,7 @@ def patch_class(clas: type, cfg: Config): declared_location=Location( file=inspect.getfile(clas), line_no=inspect.getsourcelines(clas)[1], - source_line="".join(inspect.getsourcelines(clas)[0]), + line_span=len(inspect.getsourcelines(clas)[0]), ), checkedpkgprefixes=cfg.checkedprefixes) except (TypeError, OSError) as e: # Built in types ctx = DefaultCreationContext( @@ -36,7 +36,7 @@ def patch_class(clas: type, cfg: Config): declared_location=Location( file="", line_no=0, - source_line="", + line_span=1 ), checkedpkgprefixes=cfg.checkedprefixes) setattr(clas, '__patched', True) From 0774f6ca603d55a5c8612931400735499f7dad18 Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 18:58:21 +0200 Subject: [PATCH 05/20] add return traces to import hook --- python/deps/untypy/untypy/__init__.py | 15 ++++++++++----- python/deps/untypy/untypy/patching/import_hook.py | 8 ++++---- python/deps/untypy/untypy/util/return_traces.py | 7 ++++--- .../untypy/untypy/util/tranformer_combinator.py | 11 +++++++++++ 4 files changed, 29 insertions(+), 12 deletions(-) create mode 100644 python/deps/untypy/untypy/util/tranformer_combinator.py diff --git a/python/deps/untypy/untypy/__init__.py b/python/deps/untypy/untypy/__init__.py index e96f0051..03c74c3e 100644 --- a/python/deps/untypy/untypy/__init__.py +++ b/python/deps/untypy/untypy/__init__.py @@ -10,10 +10,15 @@ from .patching.import_hook import install_import_hook from .util.condition import FunctionCondition from .util.return_traces import ReturnTracesTransformer, before_return, GlobalReturnTraceManager +from .util.tranformer_combinator import TransformerCombinator GlobalConfig = DefaultConfig _before_return = before_return +_importhook_transformer_builder = lambda path, file: TransformerCombinator(UntypyAstTransformer(), + ReturnTracesTransformer(file)) + + def just_install_hook(prefixes=[]): def predicate(module_name): for p in prefixes: @@ -23,7 +28,7 @@ def predicate(module_name): return True return False - install_import_hook(predicate, lambda path: UntypyAstTransformer()) # TODO: ReturnTracesTransformer FIX ME!!! + install_import_hook(predicate, _importhook_transformer_builder) def transform_tree(tree, file): @@ -64,9 +69,9 @@ def predicate(module_name): else: raise AssertionError("You cannot run 'untypy.enable()' twice!") - transformer = lambda path: UntypyAstTransformer() + transformer = _importhook_transformer_builder install_import_hook(predicate, transformer) - _exec_module_patched(root, exit_after, transformer(caller.__name__.split("."))) + _exec_module_patched(root, exit_after, transformer(caller.__name__.split("."), caller.__file__)) def enable_on_imports(*prefixes): @@ -86,7 +91,7 @@ def predicate(module_name: str): else: return False - transformer = lambda path: UntypyAstImportTransformer(predicate, path) + transformer = _importhook_transformer_builder install_import_hook(predicate, transformer) _exec_module_patched(caller, True, transformer(caller.__name__.split("."))) @@ -102,7 +107,7 @@ def _exec_module_patched(mod: ModuleType, exit_after: bool, transformer: ast.Nod "\tuntypy.enable()") transformer.visit(tree) - ReturnTracesTransformer(lambda r: GlobalReturnTraceManager.next_id(r, mod.__file__)).visit(tree) + ReturnTracesTransformer(mod.__file__).visit(tree) ast.fix_missing_locations(tree) patched_mod = compile(tree, mod.__file__, 'exec', dont_inherit=True, optimize=-1) stack = list(map(lambda s: s.frame, inspect.stack())) diff --git a/python/deps/untypy/untypy/patching/import_hook.py b/python/deps/untypy/untypy/patching/import_hook.py index a1768696..c9d8cc6a 100644 --- a/python/deps/untypy/untypy/patching/import_hook.py +++ b/python/deps/untypy/untypy/patching/import_hook.py @@ -6,7 +6,7 @@ from importlib.util import decode_source -def install_import_hook(should_patch_predicate: Callable[[str], bool], +def install_import_hook(should_patch_predicate: Callable[[str, str], bool], transformer: Callable[[str], ast.NodeTransformer]): import sys @@ -20,7 +20,7 @@ def install_import_hook(should_patch_predicate: Callable[[str], bool], class UntypyFinder(MetaPathFinder): - def __init__(self, inner_finder: MetaPathFinder, should_patch_predicate: Callable[[str], bool], + def __init__(self, inner_finder: MetaPathFinder, should_patch_predicate: Callable[[str, str], bool], transformer: Callable[[str], ast.NodeTransformer]): self.inner_finder = inner_finder self.should_patch_predicate = should_patch_predicate @@ -41,7 +41,7 @@ def should_instrument(self, module_name: str) -> bool: class UntypyLoader(SourceFileLoader): - def __init__(self, fullname, path, transformer: Callable[[str], ast.NodeTransformer]): + def __init__(self, fullname, path, transformer: Callable[[str, str], ast.NodeTransformer]): super().__init__(fullname, path) self.transformer = transformer @@ -49,7 +49,7 @@ def source_to_code(self, data, path, *, _optimize=-1): source = decode_source(data) tree = compile(source, path, 'exec', ast.PyCF_ONLY_AST, dont_inherit=True, optimize=_optimize) - self.transformer(self.name.split('.')).visit(tree) + self.transformer(self.name.split('.'), self.path).visit(tree) ast.fix_missing_locations(tree) return compile(tree, path, 'exec', dont_inherit=True, optimize=_optimize) diff --git a/python/deps/untypy/untypy/util/return_traces.py b/python/deps/untypy/untypy/util/return_traces.py index 6b78d5af..d9ef98cc 100644 --- a/python/deps/untypy/untypy/util/return_traces.py +++ b/python/deps/untypy/untypy/util/return_traces.py @@ -30,8 +30,9 @@ def get_last_return() -> (str, int): class ReturnTracesTransformer(ast.NodeTransformer): - def __init__(self, registry: Callable[[ast.Return], int]): - self.registry = registry + def __init__(self, file: str, manager = GlobalReturnTraceManager): + self.file = file + self.manager = manager def generic_visit(self, node) -> Any: # See https://docs.python.org/3/library/ast.html @@ -57,7 +58,7 @@ def generic_visit(self, node) -> Any: for index, reti in inserts: n = ast.Expr(value=ast.Call( func=ast.Attribute(value=ast.Name(id='untypy', ctx=ast.Load()), attr='_before_return', ctx=ast.Load()), - args=[ast.Constant(value=self.registry(reti))], + args=[ast.Constant(value=self.manager.next_id(reti, self.file))], keywords=[]) ) statements.insert(index, n) diff --git a/python/deps/untypy/untypy/util/tranformer_combinator.py b/python/deps/untypy/untypy/util/tranformer_combinator.py new file mode 100644 index 00000000..c86de398 --- /dev/null +++ b/python/deps/untypy/untypy/util/tranformer_combinator.py @@ -0,0 +1,11 @@ +import ast + + +class TransformerCombinator(ast.NodeTransformer): + + def __init__(self, *inner: ast.NodeTransformer): + self.inner = inner + + def visit(self, node): + for inner in self.inner: + inner.visit(node) From 502fc2017a316d834c99d5421f0126e54919dc77 Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 19:07:00 +0200 Subject: [PATCH 06/20] update runner for new tranform_tree --- python/src/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/runner.py b/python/src/runner.py index f74a4014..e28abb7b 100644 --- a/python/src/runner.py +++ b/python/src/runner.py @@ -281,7 +281,7 @@ def runCode(fileToRun, globals, args, useUntypy=True): verbose(f"transforming {fileToRun} for typechecking") tree = compile(codeTxt, fileToRun, 'exec', flags=(flags | ast.PyCF_ONLY_AST), dont_inherit=True, optimize=-1) - untypy.transform_tree(tree) + untypy.transform_tree(tree, os.path.abspath(fileToRun)) verbose(f'done with transformation of {fileToRun}') code = tree else: From f39871b4981c17196bb3256ac5af4ba461f55a6d Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 22:45:26 +0200 Subject: [PATCH 07/20] add unit test for return traces ast transformer --- python/deps/untypy/test/util_test/__init__.py | 0 .../test/util_test/test_return_traces.py | 33 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 python/deps/untypy/test/util_test/__init__.py create mode 100644 python/deps/untypy/test/util_test/test_return_traces.py diff --git a/python/deps/untypy/test/util_test/__init__.py b/python/deps/untypy/test/util_test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/deps/untypy/test/util_test/test_return_traces.py b/python/deps/untypy/test/util_test/test_return_traces.py new file mode 100644 index 00000000..eb110ed8 --- /dev/null +++ b/python/deps/untypy/test/util_test/test_return_traces.py @@ -0,0 +1,33 @@ +import ast +import unittest + +from untypy.util.return_traces import ReturnTraceManager, ReturnTracesTransformer + + +class TestAstTransform(unittest.TestCase): + + def test_ast_transform(self): + src = """ +def foo(flag: bool) -> int: + print('Hello World') + if flag: + return 1 + else: + return 'you stupid' + """ + target = """ +def foo(flag: bool) -> int: + print('Hello World') + if flag: + untypy._before_return(0) + return 1 + else: + untypy._before_return(1) + return 'you stupid' + """ + + tree = ast.parse(src) + ReturnTracesTransformer("", ReturnTraceManager()).visit(tree) + ast.fix_missing_locations(tree) + self.assertEqual(ast.unparse(tree).strip(), target.strip()) + From fb599d56b43c4b0aac9af6bab60ef416f4af3e81 Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 22:47:24 +0200 Subject: [PATCH 08/20] unit test ReturnTraceManager --- python/deps/untypy/test/util_test/test_return_traces.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/deps/untypy/test/util_test/test_return_traces.py b/python/deps/untypy/test/util_test/test_return_traces.py index eb110ed8..5c9e3337 100644 --- a/python/deps/untypy/test/util_test/test_return_traces.py +++ b/python/deps/untypy/test/util_test/test_return_traces.py @@ -27,7 +27,10 @@ def foo(flag: bool) -> int: """ tree = ast.parse(src) - ReturnTracesTransformer("", ReturnTraceManager()).visit(tree) + mgr = ReturnTraceManager() + ReturnTracesTransformer("", mgr).visit(tree) ast.fix_missing_locations(tree) self.assertEqual(ast.unparse(tree).strip(), target.strip()) + self.assertEqual(mgr.get(0), ("", 5)) + self.assertEqual(mgr.get(1), ("", 7)) From ef9829c9ac017183b955ab092779989d58da03e1 Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 22:51:48 +0200 Subject: [PATCH 09/20] fix #26 use abspath on runner. Relative paths are already used, due to refactor for #15 --- python/src/runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/src/runner.py b/python/src/runner.py index e28abb7b..a46f020e 100644 --- a/python/src/runner.py +++ b/python/src/runner.py @@ -269,6 +269,7 @@ def __exit__(self, exc_type, value, traceback): def runCode(fileToRun, globals, args, useUntypy=True): localDir = os.path.dirname(fileToRun) + fileToRun = os.path.abspath(fileToRun) with RunSetup(localDir): with open(fileToRun) as f: flags = 0 | anns.compiler_flag @@ -281,7 +282,7 @@ def runCode(fileToRun, globals, args, useUntypy=True): verbose(f"transforming {fileToRun} for typechecking") tree = compile(codeTxt, fileToRun, 'exec', flags=(flags | ast.PyCF_ONLY_AST), dont_inherit=True, optimize=-1) - untypy.transform_tree(tree, os.path.abspath(fileToRun)) + untypy.transform_tree(tree, fileToRun) verbose(f'done with transformation of {fileToRun}') code = tree else: From 3371ca99f3eb90a4351774b491f6a5bdbee88a0d Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 23:03:39 +0200 Subject: [PATCH 10/20] add rudimentary documentation --- python/deps/untypy/untypy/__init__.py | 8 +++++++- python/deps/untypy/untypy/error.py | 7 ++++++- python/deps/untypy/untypy/util/__init__.py | 2 +- python/deps/untypy/untypy/util/return_traces.py | 10 +++++++++- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/python/deps/untypy/untypy/__init__.py b/python/deps/untypy/untypy/__init__.py index 03c74c3e..d0fcfab2 100644 --- a/python/deps/untypy/untypy/__init__.py +++ b/python/deps/untypy/untypy/__init__.py @@ -13,12 +13,18 @@ from .util.tranformer_combinator import TransformerCombinator GlobalConfig = DefaultConfig + +""" +This function is called before any return statement, to store which was the last return. +For this the AST is transformed using ReturnTracesTransformer. +Must be in untypy so it can be used in transformed module. +Must also be in other module, so it can be used from inside (No circular imports). +""" _before_return = before_return _importhook_transformer_builder = lambda path, file: TransformerCombinator(UntypyAstTransformer(), ReturnTracesTransformer(file)) - def just_install_hook(prefixes=[]): def predicate(module_name): for p in prefixes: diff --git a/python/deps/untypy/untypy/error.py b/python/deps/untypy/untypy/error.py index 74d0da62..72b7b0fb 100644 --- a/python/deps/untypy/untypy/error.py +++ b/python/deps/untypy/untypy/error.py @@ -112,7 +112,12 @@ def from_stack(stack) -> Location: line_span=1 ) - def mark(self, reti_loc): + def narrow_in_span(self, reti_loc : Tuple[str, int]): + """ + Use new Location if inside of span of this Location + :param reti_loc: filename and line_no + :return: a new Location, else self + """ file, line = reti_loc if self.file == file and line in range(self.line_no, self.line_no + self.line_span): return Location( diff --git a/python/deps/untypy/untypy/util/__init__.py b/python/deps/untypy/untypy/util/__init__.py index 9c76dc31..591ce185 100644 --- a/python/deps/untypy/untypy/util/__init__.py +++ b/python/deps/untypy/untypy/util/__init__.py @@ -127,7 +127,7 @@ def wrap(self, err: UntypyTypeError) -> UntypyTypeError: declared = WrappedFunction.find_location(self.fn) responsable = declared if responsable is not None: - responsable = responsable.mark(self.reti_loc) + responsable = responsable.narrow_in_span(self.reti_loc) return err.with_frame(Frame( return_id.ty, diff --git a/python/deps/untypy/untypy/util/return_traces.py b/python/deps/untypy/untypy/util/return_traces.py index d9ef98cc..84eca8b0 100644 --- a/python/deps/untypy/untypy/util/return_traces.py +++ b/python/deps/untypy/untypy/util/return_traces.py @@ -3,6 +3,9 @@ class ReturnTraceManager: + """ + Stores file & line_no to every return idx + """ def __init__(self): self.lst = [] @@ -17,20 +20,25 @@ def get(self, idx : int) -> (str, int): GlobalReturnTraceManager = ReturnTraceManager() reti_loc: int = -1 + def before_return(idx : int): global reti_loc reti_loc = idx + def get_last_return() -> (str, int): global reti_loc if reti_loc < 0: return ("", 0) # this will never match any real location + + # Note: this location is only used if it is in the span of the located function. + # See ReturnExecutionContext return GlobalReturnTraceManager.get(reti_loc) class ReturnTracesTransformer(ast.NodeTransformer): - def __init__(self, file: str, manager = GlobalReturnTraceManager): + def __init__(self, file: str, manager=GlobalReturnTraceManager): self.file = file self.manager = manager From 1b472b9009d1d9c704ad93f7ec760dd89f818bd7 Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 23:13:58 +0200 Subject: [PATCH 11/20] Revert "fix #26 use abspath on runner." - Unittest issues This reverts commit ef9829c9ac017183b955ab092779989d58da03e1. --- python/src/runner.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/src/runner.py b/python/src/runner.py index a46f020e..e28abb7b 100644 --- a/python/src/runner.py +++ b/python/src/runner.py @@ -269,7 +269,6 @@ def __exit__(self, exc_type, value, traceback): def runCode(fileToRun, globals, args, useUntypy=True): localDir = os.path.dirname(fileToRun) - fileToRun = os.path.abspath(fileToRun) with RunSetup(localDir): with open(fileToRun) as f: flags = 0 | anns.compiler_flag @@ -282,7 +281,7 @@ def runCode(fileToRun, globals, args, useUntypy=True): verbose(f"transforming {fileToRun} for typechecking") tree = compile(codeTxt, fileToRun, 'exec', flags=(flags | ast.PyCF_ONLY_AST), dont_inherit=True, optimize=-1) - untypy.transform_tree(tree, fileToRun) + untypy.transform_tree(tree, os.path.abspath(fileToRun)) verbose(f'done with transformation of {fileToRun}') code = tree else: From 622bb4a4a74bb199dd16dec8ee6dd0474fb721b8 Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 23:25:58 +0200 Subject: [PATCH 12/20] fix "just_transform" in REPL --- python/src/runner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/src/runner.py b/python/src/runner.py index e28abb7b..af3575c2 100644 --- a/python/src/runner.py +++ b/python/src/runner.py @@ -481,10 +481,12 @@ def runsource(self, source, filename="", symbol="single"): if code is None: return True try: - ast = untypy.just_transform("\n".join(self.buffer), filename, symbol) + import ast + tree = compile("\n".join(self.buffer), filename, symbol, flags=ast.PyCF_ONLY_AST, dont_inherit=True, optimize=-1) + tree = untypy.transform_tree(tree, filename) code = compile(ast, filename, symbol) except Exception as e: - if e.text == "": + if hasattr(e, "text") and e.text == "": pass else: traceback.print_tb(e.__traceback__) From dc859b725869e5226dcf82cbbef7bdc780f271bb Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 23:26:45 +0200 Subject: [PATCH 13/20] adjust error messages to new location output --- python/test-data/declared-at-missing.err | 10 ++++---- python/test-data/testForwardRef2.err | 5 +++- python/test-data/testForwardRef4.err | 4 +++- python/test-data/testTypes1.err | 8 +++++-- python/test-data/testTypes2.err | 8 +++++-- python/test-data/testTypesCollections1.err | 11 +++++++-- python/test-data/testTypesCollections2.err | 10 ++++---- python/test-data/testTypesCollections4.err | 23 ++++++++++++------ python/test-data/testTypesHigherOrderFuns.err | 10 ++++---- python/test-data/testTypesProtos1.err | 24 +++++++++++++++---- python/test-data/testTypesProtos3.err | 9 +++++-- python/test-data/testTypesSequence1.err | 8 +++++-- python/test-data/testTypesSequence2.err | 8 +++++-- python/test-data/testTypesTuple1.err | 9 +++++-- 14 files changed, 107 insertions(+), 40 deletions(-) diff --git a/python/test-data/declared-at-missing.err b/python/test-data/declared-at-missing.err index 6bf0db62..cda1c6ae 100644 --- a/python/test-data/declared-at-missing.err +++ b/python/test-data/declared-at-missing.err @@ -5,11 +5,13 @@ expected: value of type CourseM context: Semester(degreeProgram: str, semester: str, courses: tuple[CourseM, ...]) -> Self ^^^^^^^ declared at: test-data/declared-at-missing.py:15 - 15 | @record + 13 | students: tuple[str, ...] + 14 | + 15 > @record 16 | class Semester: 17 | degreeProgram: str - 18 | semester: str - 19 | courses: tuple[CourseM, ...] caused by: test-data/declared-at-missing.py:22 - 22 | semester1_2020 = Semester('AKI', '1. Semester 2020/21', (prog1, )) + 20 | + 21 | prog1 = Course('Programmierung 1', 'Wehr', ()) + 22 > semester1_2020 = Semester('AKI', '1. Semester 2020/21', (prog1, )) diff --git a/python/test-data/testForwardRef2.err b/python/test-data/testForwardRef2.err index aacae147..eaf882e8 100644 --- a/python/test-data/testForwardRef2.err +++ b/python/test-data/testForwardRef2.err @@ -2,5 +2,8 @@ untypy.error.UntypyNameError: The name 'Foo' is not defined. Type annotation of function 'Test.__init__' is not resoveable. Did you forget importing this type? test-data/testForwardRef2.py:2 - 2 | def __init__(self, foo: 'Foo'): + 1 | class Test: + 2 > def __init__(self, foo: 'Foo'): 3 | pass + 4 | def __repr__(self): + 5 | return 'Test' diff --git a/python/test-data/testForwardRef4.err b/python/test-data/testForwardRef4.err index 84f55d7e..ad96e9d6 100644 --- a/python/test-data/testForwardRef4.err +++ b/python/test-data/testForwardRef4.err @@ -2,6 +2,8 @@ untypy.error.UntypyNameError: The name 'FooX' is not defined. Type annotation of Class 'Test' is not resoveable. Did you forget importing this type? test-data/testForwardRef4.py:3 - 3 | @record + 1 | from wypp import * + 2 | + 3 > @record 4 | class Test: 5 | foo: 'FooX' diff --git a/python/test-data/testTypes1.err b/python/test-data/testTypes1.err index d52cba13..8d34bdba 100644 --- a/python/test-data/testTypes1.err +++ b/python/test-data/testTypes1.err @@ -5,8 +5,12 @@ expected: value of type int context: inc(x: int) -> int ^^^ declared at: test-data/testTypes1.py:1 - 1 | def inc(x: int) -> int: + 1 > def inc(x: int) -> int: 2 | return x + 1 + 3 | + 4 | inc("1") caused by: test-data/testTypes1.py:4 - 4 | inc("1") + 2 | return x + 1 + 3 | + 4 > inc("1") diff --git a/python/test-data/testTypes2.err b/python/test-data/testTypes2.err index 6a5fed85..c2eec48d 100644 --- a/python/test-data/testTypes2.err +++ b/python/test-data/testTypes2.err @@ -5,8 +5,12 @@ expected: value of type int context: inc(x: int) -> int ^^^ declared at: test-data/testTypes2.py:1 - 1 | def inc(x: int) -> int: + 1 > def inc(x: int) -> int: 2 | return x + 3 | + 4 | inc("1") caused by: test-data/testTypes2.py:4 - 4 | inc("1") + 2 | return x + 3 | + 4 > inc("1") diff --git a/python/test-data/testTypesCollections1.err b/python/test-data/testTypesCollections1.err index 2606513a..4978ead6 100644 --- a/python/test-data/testTypesCollections1.err +++ b/python/test-data/testTypesCollections1.err @@ -5,8 +5,15 @@ expected: value of type int context: list[int] ^^^ declared at: test-data/testTypesCollections1.py:3 - 3 | def appendSomething(l: list[int]) -> None: + 1 | from wypp import * + 2 | + 3 > def appendSomething(l: list[int]) -> None: 4 | l.append("foo") + 5 | caused by: test-data/testTypesCollections1.py:4 - 4 | l.append("foo") + 2 | + 3 | def appendSomething(l: list[int]) -> None: + 4 > l.append("foo") + 5 | + 6 | l = [1,2,3] diff --git a/python/test-data/testTypesCollections2.err b/python/test-data/testTypesCollections2.err index 1f7b2b48..3edb9f86 100644 --- a/python/test-data/testTypesCollections2.err +++ b/python/test-data/testTypesCollections2.err @@ -5,11 +5,13 @@ expected: value of type str context: foo(l: list[Callable[[], str]]) -> list[str] ^^^ declared at: test-data/testTypesCollections2.py:3 - 3 | def foo(l: list[Callable[[], str]]) -> list[str]: + 1 | from wypp import * + 2 | + 3 > def foo(l: list[Callable[[], str]]) -> list[str]: 4 | res = [] 5 | for f in l: - 6 | res.append(f()) - 7 | return res caused by: test-data/testTypesCollections2.py:10 - 10 | foo([lambda: "1", lambda: 42]) # error because the 2nd functions returns an int + 8 | + 9 | print(foo([lambda: "1", lambda: "2"])) + 10 > foo([lambda: "1", lambda: 42]) # error because the 2nd functions returns an int diff --git a/python/test-data/testTypesCollections4.err b/python/test-data/testTypesCollections4.err index b8d0192d..1dfee0f1 100644 --- a/python/test-data/testTypesCollections4.err +++ b/python/test-data/testTypesCollections4.err @@ -5,16 +5,25 @@ expected: value of type str context: foo(l: list[Callable[[], str]]) -> list[str] ^^^ declared at: test-data/testTypesCollections4.py:4 - 4 | l.append(lambda: 42) # error -declared at: test-data/testTypesCollections4.py:3 + 2 | 3 | def foo(l: list[Callable[[], str]]) -> list[str]: - 4 | l.append(lambda: 42) # error + 4 > l.append(lambda: 42) # error 5 | res = [] 6 | for f in l: - 7 | res.append(f()) - | ... +declared at: test-data/testTypesCollections4.py:3 + 1 | from wypp import * + 2 | + 3 > def foo(l: list[Callable[[], str]]) -> list[str]: + 4 | l.append(lambda: 42) # error + 5 | res = [] caused by: test-data/testTypesCollections4.py:4 - 4 | l.append(lambda: 42) # error + 2 | + 3 | def foo(l: list[Callable[[], str]]) -> list[str]: + 4 > l.append(lambda: 42) # error + 5 | res = [] + 6 | for f in l: caused by: test-data/testTypesCollections4.py:10 - 10 | foo([]) + 8 | return res + 9 | + 10 > foo([]) diff --git a/python/test-data/testTypesHigherOrderFuns.err b/python/test-data/testTypesHigherOrderFuns.err index 4b406bdb..614ba240 100644 --- a/python/test-data/testTypesHigherOrderFuns.err +++ b/python/test-data/testTypesHigherOrderFuns.err @@ -5,11 +5,13 @@ expected: value of type int context: map(container: Iterable[str], fun: Callable[[str], int]) -> list[int] ^^^ declared at: test-data/testTypesHigherOrderFuns.py:3 - 3 | def map(container: Iterable[str], fun: Callable[[str], int]) -> list[int]: + 1 | from wypp import * + 2 | + 3 > def map(container: Iterable[str], fun: Callable[[str], int]) -> list[int]: 4 | res = [] 5 | for x in container: - 6 | res.append(fun(x)) - 7 | return res caused by: test-data/testTypesHigherOrderFuns.py:10 - 10 | map(["hello", "1"], lambda x: x) + 8 | + 9 | print(map(["hello", "1"], len)) + 10 > map(["hello", "1"], lambda x: x) diff --git a/python/test-data/testTypesProtos1.err b/python/test-data/testTypesProtos1.err index eaf43a4e..647ae296 100644 --- a/python/test-data/testTypesProtos1.err +++ b/python/test-data/testTypesProtos1.err @@ -7,11 +7,16 @@ expected: value of type Animal context: doSomething(a: Animal) -> None ^^^^^^ declared at: test-data/testTypesProtos1.py:18 - 18 | def doSomething(a: Animal) -> None: + 16 | return "" + 17 | + 18 > def doSomething(a: Animal) -> None: 19 | print(a.makeSound(3.14)) + 20 | caused by: test-data/testTypesProtos1.py:21 - 21 | doSomething(Dog()) + 19 | print(a.makeSound(3.14)) + 20 | + 21 > doSomething(Dog()) The argument 'loadness' of method 'makeSound' violates the protocol 'Animal'. The annotation 'int' is incompatible with the protocol's annotation 'float' @@ -23,12 +28,21 @@ expected: value of type int context: makeSound(self: Self, loadness: int) -> str ^^^ declared at: test-data/testTypesProtos1.py:13 - 13 | def makeSound(self, loadness: int) -> str: + 11 | class Dog: + 12 | # incorrect implementation of the Animal protocol + 13 > def makeSound(self, loadness: int) -> str: 14 | return f"{loadness} wuffs" + 15 | def __repr__(self): declared at: test-data/testTypesProtos1.py:8 - 8 | def makeSound(self, loadness: float) -> str: + 6 | class Animal(Protocol): + 7 | @abc.abstractmethod + 8 > def makeSound(self, loadness: float) -> str: 9 | pass + 10 | caused by: test-data/testTypesProtos1.py:13 - 13 | def makeSound(self, loadness: int) -> str: + 11 | class Dog: + 12 | # incorrect implementation of the Animal protocol + 13 > def makeSound(self, loadness: int) -> str: 14 | return f"{loadness} wuffs" + 15 | def __repr__(self): diff --git a/python/test-data/testTypesProtos3.err b/python/test-data/testTypesProtos3.err index 4be46a5f..fde5a9cc 100644 --- a/python/test-data/testTypesProtos3.err +++ b/python/test-data/testTypesProtos3.err @@ -7,8 +7,13 @@ expected: value of type Animal context: doSomething(a: Animal) -> None ^^^^^^ declared at: test-data/testTypesProtos3.py:16 - 16 | def doSomething(a: Animal) -> None: + 14 | return f"{loadness} wuffs" + 15 | + 16 > def doSomething(a: Animal) -> None: 17 | print(a.makeSound(3.14)) + 18 | caused by: test-data/testTypesProtos3.py:19 - 19 | doSomething(Dog()) + 17 | print(a.makeSound(3.14)) + 18 | + 19 > doSomething(Dog()) diff --git a/python/test-data/testTypesSequence1.err b/python/test-data/testTypesSequence1.err index eb475241..328fdc7d 100644 --- a/python/test-data/testTypesSequence1.err +++ b/python/test-data/testTypesSequence1.err @@ -5,9 +5,13 @@ expected: value of type Sequence context: foo(seq: Sequence) -> None ^^^^^^^^ declared at: test-data/testTypesSequence1.py:3 - 3 | def foo(seq: Sequence) -> None: + 1 | from wypp import * + 2 | + 3 > def foo(seq: Sequence) -> None: 4 | print(seq) 5 | pass caused by: test-data/testTypesSequence1.py:10 - 10 | foo(1) # should fail + 8 | foo( ("bar", "baz") ) + 9 | foo("Hello!") + 10 > foo(1) # should fail diff --git a/python/test-data/testTypesSequence2.err b/python/test-data/testTypesSequence2.err index abccc73a..a4256195 100644 --- a/python/test-data/testTypesSequence2.err +++ b/python/test-data/testTypesSequence2.err @@ -5,9 +5,13 @@ expected: value of type Sequence[int] context: foo(seq: Sequence[int]) -> None ^^^^^^^^^^^^^ declared at: test-data/testTypesSequence2.py:3 - 3 | def foo(seq: Sequence[int]) -> None: + 1 | from wypp import * + 2 | + 3 > def foo(seq: Sequence[int]) -> None: 4 | print(seq) 5 | pass caused by: test-data/testTypesSequence2.py:9 - 9 | foo("Hello!") # should fail + 7 | foo([1,2,3]) + 8 | foo( (4,5) ) + 9 > foo("Hello!") # should fail diff --git a/python/test-data/testTypesTuple1.err b/python/test-data/testTypesTuple1.err index c9ae74fb..ef242de6 100644 --- a/python/test-data/testTypesTuple1.err +++ b/python/test-data/testTypesTuple1.err @@ -5,8 +5,13 @@ expected: value of type tuple[int, ...] context: foo(l: tuple[int, ...]) -> int ^^^^^^^^^^^^^^^ declared at: test-data/testTypesTuple1.py:1 - 1 | def foo(l: tuple[int, ...]) -> int: + 1 > def foo(l: tuple[int, ...]) -> int: 2 | return len(l) + 3 | + 4 | print(foo((1,2,3))) + 5 | foo(1) caused by: test-data/testTypesTuple1.py:5 - 5 | foo(1) + 3 | + 4 | print(foo((1,2,3))) + 5 > foo(1) From 970fb8db1b33ae87d2b565fbc84883476590981d Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 23:41:11 +0200 Subject: [PATCH 14/20] fix REPL II --- python/src/runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/src/runner.py b/python/src/runner.py index af3575c2..dadb7009 100644 --- a/python/src/runner.py +++ b/python/src/runner.py @@ -483,10 +483,10 @@ def runsource(self, source, filename="", symbol="single"): try: import ast tree = compile("\n".join(self.buffer), filename, symbol, flags=ast.PyCF_ONLY_AST, dont_inherit=True, optimize=-1) - tree = untypy.transform_tree(tree, filename) - code = compile(ast, filename, symbol) + untypy.transform_tree(tree, filename) + code = compile(tree, filename, symbol) except Exception as e: - if hasattr(e, "text") and e.text == "": + if hasattr(e, 'text') and e.text == "": pass else: traceback.print_tb(e.__traceback__) From a9be28866cb8613d9ad9ed32328ab1c0ac29b35f Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 23:47:12 +0200 Subject: [PATCH 15/20] fix integration test for new location format --- python/integration-tests/testIntegration.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/python/integration-tests/testIntegration.py b/python/integration-tests/testIntegration.py index f344ee53..18edf370 100644 --- a/python/integration-tests/testIntegration.py +++ b/python/integration-tests/testIntegration.py @@ -59,12 +59,14 @@ def test_recordFail1(self): context: Person(name: str, age: int) -> Self ^^^ declared at: test-data/typeRecords.py:3 - 3 | @record + 1 | from wypp import * + 2 | + 3 > @record 4 | class Person: 5 | name: str - 6 | age: int -caused by: :1""" +caused by: :1 + 1 | """ self.assertEqual(expected, '\n'.join(out)) def test_recordFail2(self): @@ -153,10 +155,11 @@ def test_types2(self): context: inc(x: int) -> int ^^^ declared at: test-data/testTypesInteractive.py:1 - 1 | def inc(x: int) -> int: + 1 > def inc(x: int) -> int: 2 | return x + 1 -caused by: :1""" +caused by: :1 + 1 | """ self.assertEqual(expected, out) def test_types3(self): From 94d4e17ce06661647bf226300c8b0de85b4bb493 Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 23:58:23 +0200 Subject: [PATCH 16/20] add missing argument in "enable_on_imports" --- python/deps/untypy/untypy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/deps/untypy/untypy/__init__.py b/python/deps/untypy/untypy/__init__.py index d0fcfab2..7d047af4 100644 --- a/python/deps/untypy/untypy/__init__.py +++ b/python/deps/untypy/untypy/__init__.py @@ -99,7 +99,7 @@ def predicate(module_name: str): transformer = _importhook_transformer_builder install_import_hook(predicate, transformer) - _exec_module_patched(caller, True, transformer(caller.__name__.split("."))) + _exec_module_patched(caller, True, transformer(caller.__name__.split("."), caller.__file__)) def _exec_module_patched(mod: ModuleType, exit_after: bool, transformer: ast.NodeTransformer): From cf7e403b64133f0b71f8fe921de619ffdea67a18 Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Wed, 29 Sep 2021 23:58:56 +0200 Subject: [PATCH 17/20] add test for #15 --- python/fileTests | 2 +- python/test-data/testTypesReturn.err | 19 +++++++++++++++++++ python/test-data/testTypesReturn.out | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 python/test-data/testTypesReturn.err create mode 100644 python/test-data/testTypesReturn.out diff --git a/python/fileTests b/python/fileTests index 0231b5ab..644e43d9 100755 --- a/python/fileTests +++ b/python/fileTests @@ -137,7 +137,7 @@ checkWithOutputAux yes 0 test-data/testForwardRef1.py checkWithOutputAux yes 1 test-data/testForwardRef2.py checkWithOutputAux yes 0 test-data/testForwardRef3.py checkWithOutputAux yes 1 test-data/testForwardRef4.py -# checkWithOutputAux yes 1 test-data/testTypesReturn.py See #15 +checkWithOutputAux yes 1 test-data/testTypesReturn.py checkWithOutputAux yes 1 test-data/testTypesSequence1.py checkWithOutputAux yes 1 test-data/testTypesSequence2.py checkWithOutputAux yes 1 test-data/testTypesTuple1.py diff --git a/python/test-data/testTypesReturn.err b/python/test-data/testTypesReturn.err new file mode 100644 index 00000000..69329736 --- /dev/null +++ b/python/test-data/testTypesReturn.err @@ -0,0 +1,19 @@ +untypy.error.UntypyTypeError +given: 'you stupid' +expected: value of type int + +context: foo(flag: bool) -> int + ^^^ +declared at: test-data/testTypesReturn.py:1 + 1 > def foo(flag: bool) -> int: + 2 | print('Hello World') + 3 | if flag: + 4 | return 1 + 5 | else: + +caused by: test-data/testTypesReturn.py:6 + 4 | return 1 + 5 | else: + 6 > return 'you stupid' + 7 | + 8 | foo(False) diff --git a/python/test-data/testTypesReturn.out b/python/test-data/testTypesReturn.out new file mode 100644 index 00000000..557db03d --- /dev/null +++ b/python/test-data/testTypesReturn.out @@ -0,0 +1 @@ +Hello World From 6e2c1a56cf73d401bb110e2c94bba08a92ead496 Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Thu, 30 Sep 2021 00:15:57 +0200 Subject: [PATCH 18/20] use abspath - needed matching of return traces --- python/src/runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/src/runner.py b/python/src/runner.py index dadb7009..a3e460dd 100644 --- a/python/src/runner.py +++ b/python/src/runner.py @@ -269,6 +269,7 @@ def __exit__(self, exc_type, value, traceback): def runCode(fileToRun, globals, args, useUntypy=True): localDir = os.path.dirname(fileToRun) + fileToRun = os.path.abspath(fileToRun) with RunSetup(localDir): with open(fileToRun) as f: flags = 0 | anns.compiler_flag @@ -281,7 +282,7 @@ def runCode(fileToRun, globals, args, useUntypy=True): verbose(f"transforming {fileToRun} for typechecking") tree = compile(codeTxt, fileToRun, 'exec', flags=(flags | ast.PyCF_ONLY_AST), dont_inherit=True, optimize=-1) - untypy.transform_tree(tree, os.path.abspath(fileToRun)) + untypy.transform_tree(tree, fileToRun) verbose(f'done with transformation of {fileToRun}') code = tree else: From 0fa302d5fd59118a6f0a5903454370463a633dc8 Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Thu, 30 Sep 2021 00:33:12 +0200 Subject: [PATCH 19/20] dont use abspath - breaks return traces --- python/src/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/runner.py b/python/src/runner.py index a3e460dd..d0d34eb8 100644 --- a/python/src/runner.py +++ b/python/src/runner.py @@ -269,7 +269,7 @@ def __exit__(self, exc_type, value, traceback): def runCode(fileToRun, globals, args, useUntypy=True): localDir = os.path.dirname(fileToRun) - fileToRun = os.path.abspath(fileToRun) + with RunSetup(localDir): with open(fileToRun) as f: flags = 0 | anns.compiler_flag From 439b76eff84270658c928f784fd49fcc302eeb3a Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Thu, 30 Sep 2021 00:43:14 +0200 Subject: [PATCH 20/20] fix testForwardRef*-Test after merge --- python/test-data/testForwardRef2.err | 3 ++- python/test-data/testForwardRef4.err | 14 +++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/python/test-data/testForwardRef2.err b/python/test-data/testForwardRef2.err index 467f59a4..9fa569ae 100644 --- a/python/test-data/testForwardRef2.err +++ b/python/test-data/testForwardRef2.err @@ -1,7 +1,8 @@ untypy.error.UntypyNameError: name 'Foo' is not defined. Type annotation of function 'Test.__init__' could not be resolved. declared at: test-data/testForwardRef2.py:2 - 2 | def __init__(self, foo: 'Foo'): + 1 | class Test: + 2 > def __init__(self, foo: 'Foo'): 3 | pass 4 | def __repr__(self): 5 | return 'Test' diff --git a/python/test-data/testForwardRef4.err b/python/test-data/testForwardRef4.err index 8b1a7124..d17f482c 100644 --- a/python/test-data/testForwardRef4.err +++ b/python/test-data/testForwardRef4.err @@ -1,16 +1,8 @@ -<<<<<<< HEAD -untypy.error.UntypyNameError: The name 'FooX' is not defined. -Type annotation of Class 'Test' is not resoveable. -Did you forget importing this type? -test-data/testForwardRef4.py:3 - 1 | from wypp import * - 2 | - 3 > @record -======= untypy.error.UntypyNameError: name 'FooX' is not defined. Type annotation of class 'Test' could not be resolved. declared at: test-data/testForwardRef4.py:3 - 3 | @record ->>>>>>> master + 1 | from wypp import * + 2 | + 3 > @record 4 | class Test: 5 | foo: 'FooX'