From 0a298d8c93765d9583914ab2deeb1fc174b49db2 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 7 Jun 2022 15:40:11 -0700 Subject: [PATCH 1/4] Fail fast if module like typing not from typeshed --- mypy/build.py | 34 ++++++++++++++++++++++------------ mypy/checker.py | 2 +- mypy/errors.py | 11 ++++++----- mypy/semanal.py | 3 ++- mypy/semanal_main.py | 25 +++++++++++++++++-------- mypy/server/update.py | 2 +- mypy/util.py | 7 ++++--- test-data/unit/cmdline.test | 10 ++++++++++ 8 files changed, 63 insertions(+), 31 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index c0b9aff5ab32..6ada2f7fe8d4 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -238,6 +238,7 @@ def _build(sources: List[BuildSource], # # Ignore current directory prefix in error messages. manager = BuildManager(data_dir, search_paths, + alt_lib_path, ignore_prefix=os.getcwd(), source_set=source_set, reports=reports, @@ -631,6 +632,20 @@ def __init__(self, data_dir: str, and not has_reporters) self.fscache = fscache self.find_module_cache = FindModuleCache(self.search_paths, self.fscache, self.options) + + # Check for shadowed core library modules as soon as we have a FindModuleCache + for module in CORE_BUILTIN_MODULES: + # This module is built into Python so it doesn't exist on disk + if module == '_importlib_modulespec': + continue + path = self.find_module_cache.find_module(module) + if (not is_typeshed_file(path, options.custom_typeshed_dir) + and not is_stub_package_file(path)): + raise CompileError([ + f'mypy: "{os.path.relpath(path)}" shadows library module "{module}"', + f'note: A user-defined top-level module with name "{module}" is not supported' + ]) + self.metastore = create_metastore(options) # a mapping from source files to their corresponding shadow files @@ -2400,14 +2415,11 @@ def generate_unused_ignore_notes(self) -> None: # those errors to avoid spuriously flagging them as unused ignores. if self.meta: self.verify_dependencies(suppressed_only=True) - self.manager.errors.generate_unused_ignore_errors(self.xpath) + self.manager.errors.generate_unused_ignore_errors(self.xpath, self.options) def generate_ignore_without_code_notes(self) -> None: if self.manager.errors.is_error_code_enabled(codes.IGNORE_WITHOUT_CODE): - self.manager.errors.generate_ignore_without_code_errors( - self.xpath, - self.options.warn_unused_ignores, - ) + self.manager.errors.generate_ignore_without_code_errors(self.xpath, self.options) # Module import and diagnostic glue @@ -2485,15 +2497,13 @@ def find_module_and_diagnose(manager: BuildManager, if is_sub_path(result, dir): # Silence errors in site-package dirs and typeshed follow_imports = 'silent' - if (id in CORE_BUILTIN_MODULES - and not is_typeshed_file(result) - and not is_stub_package_file(result) - and not options.use_builtins_fixtures - and not options.custom_typeshed_dir): + """ if (id in CORE_BUILTIN_MODULES + and not is_typeshed_file(result, options.custom_typeshed_dir) + and not is_stub_package_file(result)): raise CompileError([ f'mypy: "{os.path.relpath(result)}" shadows library module "{id}"', f'note: A user-defined top-level module with name "{id}" is not supported' - ]) + ]) """ return (result, follow_imports) else: # Could not find a module. Typically the reason is a @@ -3202,7 +3212,7 @@ def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No # SemanticAnalyzerPass2.add_builtin_aliases for details. typing_mod = graph['typing'].tree assert typing_mod, "The typing module was not parsed" - mypy.semanal_main.semantic_analysis_for_scc(graph, scc, manager.errors) + mypy.semanal_main.semantic_analysis_for_scc(graph, scc, manager.errors, manager.options) # Track what modules aren't yet done so we can finish them as soon # as possible, saving memory. diff --git a/mypy/checker.py b/mypy/checker.py index 109a3b1f15d2..e836a29efa90 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -258,7 +258,7 @@ def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Option self.pass_num = 0 self.current_node_deferred = False self.is_stub = tree.is_stub - self.is_typeshed_stub = is_typeshed_file(path) + self.is_typeshed_stub = is_typeshed_file(path, options.custom_typeshed_dir) self.inferred_attribute_types = None if options.strict_optional_whitelist is None: self.suppress_none_errors = not options.show_none_errors diff --git a/mypy/errors.py b/mypy/errors.py index 0ad56b079ecc..c083f9874678 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -550,9 +550,10 @@ def clear_errors_in_targets(self, path: str, targets: Set[str]) -> None: if not has_blocker and path in self.has_blockers: self.has_blockers.remove(path) - def generate_unused_ignore_errors(self, file: str) -> None: + def generate_unused_ignore_errors(self, file: str, options: Options) -> None: + custom_typeshed_dir = options.custom_typeshed_dir ignored_lines = self.ignored_lines[file] - if not is_typeshed_file(file) and file not in self.ignored_files: + if not is_typeshed_file(file, custom_typeshed_dir) and file not in self.ignored_files: ignored_lines = self.ignored_lines[file] used_ignored_lines = self.used_ignored_lines[file] for line, ignored_codes in ignored_lines.items(): @@ -577,8 +578,8 @@ def generate_unused_ignore_errors(self, file: str) -> None: def generate_ignore_without_code_errors(self, file: str, - is_warning_unused_ignores: bool) -> None: - if is_typeshed_file(file) or file in self.ignored_files: + options: Options) -> None: + if is_typeshed_file(file, options.custom_typeshed_dir) or file in self.ignored_files: return used_ignored_lines = self.used_ignored_lines[file] @@ -595,7 +596,7 @@ def generate_ignore_without_code_errors(self, # If the ignore is itself unused and that would be warned about, let # that error stand alone - if is_warning_unused_ignores and not used_ignored_lines[line]: + if options.warn_unused_ignores and not used_ignored_lines[line]: continue codes_hint = '' diff --git a/mypy/semanal.py b/mypy/semanal.py index a49e7c23edf5..c2a5426eed79 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -573,9 +573,10 @@ def file_context(self, self.errors.set_file(file_node.path, file_node.fullname, scope=scope) self.cur_mod_node = file_node self.cur_mod_id = file_node.fullname + custom_typeshed_dir = options.custom_typeshed_dir with scope.module_scope(self.cur_mod_id): self._is_stub_file = file_node.path.lower().endswith('.pyi') - self._is_typeshed_stub_file = is_typeshed_file(file_node.path) + self._is_typeshed_stub_file = is_typeshed_file(file_node.path, custom_typeshed_dir) self.globals = file_node.names self.tvar_scope = TypeVarLikeScope() diff --git a/mypy/semanal_main.py b/mypy/semanal_main.py index bb0af8edc46f..cf7f9a1c4748 100644 --- a/mypy/semanal_main.py +++ b/mypy/semanal_main.py @@ -65,7 +65,10 @@ core_modules: Final = ['typing', 'builtins', 'abc', 'collections'] -def semantic_analysis_for_scc(graph: 'Graph', scc: List[str], errors: Errors) -> None: +def semantic_analysis_for_scc(graph: 'Graph', + scc: List[str], + errors: Errors, + options: Options) -> None: """Perform semantic analysis for all modules in a SCC (import cycle). Assume that reachability analysis has already been performed. @@ -83,7 +86,7 @@ def semantic_analysis_for_scc(graph: 'Graph', scc: List[str], errors: Errors) -> # callbacks to be required. apply_semantic_analyzer_patches(patches) # This pass might need fallbacks calculated above. - check_type_arguments(graph, scc, errors) + check_type_arguments(graph, scc, errors, options) # Run class decorator hooks (they requite complete MROs and no placeholders). apply_class_plugin_hooks(graph, scc, errors) calculate_class_properties(graph, scc, errors) @@ -134,7 +137,7 @@ def semantic_analysis_for_targets( n.node.fullname, n.node, n.active_typeinfo, patches) apply_semantic_analyzer_patches(patches) - check_type_arguments_in_targets(nodes, state, state.manager.errors) + check_type_arguments_in_targets(nodes, state) calculate_class_properties(graph, [state.id], state.manager.errors) apply_class_plugin_hooks(graph, [state.id], state.manager.errors) @@ -353,28 +356,34 @@ def semantic_analyze_target(target: str, return [], analyzer.incomplete, analyzer.progress -def check_type_arguments(graph: 'Graph', scc: List[str], errors: Errors) -> None: +def check_type_arguments(graph: 'Graph', + scc: List[str], + errors: Errors, + options: Options) -> None: for module in scc: state = graph[module] assert state.tree + typeshed_file = is_typeshed_file(state.path or '', options.custom_typeshed_dir) analyzer = TypeArgumentAnalyzer(errors, state.options, - is_typeshed_file(state.path or '')) + typeshed_file) with state.wrap_context(): with mypy.state.state.strict_optional_set(state.options.strict_optional): state.tree.accept(analyzer) -def check_type_arguments_in_targets(targets: List[FineGrainedDeferredNode], state: 'State', - errors: Errors) -> None: +def check_type_arguments_in_targets(targets: List[FineGrainedDeferredNode], + state: 'State') -> None: """Check type arguments against type variable bounds and restrictions. This mirrors the logic in check_type_arguments() except that we process only some targets. This is used in fine grained incremental mode. """ + errors = state.manager.errors + typeshed_file = is_typeshed_file(state.path or '', state.manager.options.custom_typeshed_dir) analyzer = TypeArgumentAnalyzer(errors, state.options, - is_typeshed_file(state.path or '')) + typeshed_file) with state.wrap_context(): with mypy.state.state.strict_optional_set(state.options.strict_optional): for target in targets: diff --git a/mypy/server/update.py b/mypy/server/update.py index e50bb1d158a2..9966cfbff03c 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -602,7 +602,7 @@ def restore(ids: List[str]) -> None: assert state.tree is not None, "file must be at least parsed" t0 = time.time() try: - semantic_analysis_for_scc(graph, [state.id], manager.errors) + semantic_analysis_for_scc(graph, [state.id], manager.errors, manager.options) except CompileError as err: # There was a blocking error, so module AST is incomplete. Restore old modules. restore([module]) diff --git a/mypy/util.py b/mypy/util.py index c207fb7e18cd..841c429a612f 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -749,9 +749,10 @@ def format_error( return self.style(msg, 'red', bold=True) -def is_typeshed_file(file: str) -> bool: - # gross, but no other clear way to tell - return 'typeshed' in os.path.abspath(file).split(os.sep) +def is_typeshed_file(file: str, custom_typeshed_dir: Optional[str]) -> bool: + typeshed_dir = custom_typeshed_dir or os.path.join(os.path.dirname(__file__), "typeshed") + # Check that the file prefix is in typeshed_dir + return os.path.commonpath((typeshed_dir, os.path.abspath(file))) == typeshed_dir def is_stub_package_file(file: str) -> bool: diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 86a975fc4949..1fd307e5234e 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1399,3 +1399,13 @@ b\.c \d+ # cmd: mypy --enable-incomplete-features a.py [file a.py] pass + +[case testCoreBuiltinFileNotFromTypeshed] +from typing import Union + +def foo(a: Union[int, str]) -> str: + return str +[file typing.py] +class Bar: + pass +[out] From 22bc978d1c54f04f88b86cb0a47a1e4b0b75931e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 15 Jun 2022 17:52:01 -0700 Subject: [PATCH 2/4] Fix BuildManager argument --- mypy/build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index ef34ba05e07a..f957ec63cdfb 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -211,7 +211,6 @@ def _build(sources: List[BuildSource], # # Ignore current directory prefix in error messages. manager = BuildManager(data_dir, search_paths, - alt_lib_path, ignore_prefix=os.getcwd(), source_set=source_set, reports=reports, From f12c4edb09d632d6d6392df596c96e07af0df932 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 15 Jun 2022 18:00:35 -0700 Subject: [PATCH 3/4] Fix types :) --- mypy/build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypy/build.py b/mypy/build.py index f957ec63cdfb..6a94896ad604 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -613,6 +613,10 @@ def __init__(self, data_dir: str, if module == '_importlib_modulespec': continue path = self.find_module_cache.find_module(module) + if not isinstance(path, str): + raise CompileError([ + "Failed to find builtin module {module}, perhaps typeshed is broken?", + ]) if (not is_typeshed_file(path, options.custom_typeshed_dir) and not is_stub_package_file(path)): raise CompileError([ From f935a6aae4634d43bc93ae4aaf95ecc7c9d457b8 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 15 Jun 2022 18:03:18 -0700 Subject: [PATCH 4/4] Remove spurious whitespace --- mypy/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index 6a94896ad604..60b46f398c04 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -603,7 +603,7 @@ def __init__(self, data_dir: str, or options.use_fine_grained_cache) and not has_reporters) self.fscache = fscache - + self.find_module_cache = FindModuleCache(self.search_paths, self.fscache, self.options, source_set=self.source_set)