Turn mypyc on for ~all substantial code

mypy/api.py

@@ -45,7 +45,6 @@
 import sys
 from io import StringIO
 from typing import List, Tuple, Union, TextIO, Callable
-from mypy_extensions import DefaultArg
 
 
 def _run(main_wrapper: Callable[[TextIO, TextIO], None]) -> Tuple[str, str, int]:
@@ -70,5 +69,5 @@ def run(args: List[str]) -> Tuple[str, str, int]:
 
 
 def run_dmypy(args: List[str]) -> Tuple[str, str, int]:
-    from mypy.dmypy import main
+    from mypy.dmypy.client import main
     return _run(lambda stdout, stderr: main(args))

mypy/build.py

@@ -62,8 +62,6 @@
 from mypy.typestate import TypeState, reset_global_state
 from mypy.renaming import VariableRenameVisitor
 
-from mypy.mypyc_hacks import BuildManagerBase
-
 
 # Switch to True to produce debug output related to fine-grained incremental
 # mode only that is useful during development. This produces only a subset of
@@ -455,7 +453,7 @@ def find_config_file_line_number(path: str, section: str, setting_name: str) ->
     return -1
 
 
-class BuildManager(BuildManagerBase):
+class BuildManager:
     """This class holds shared state for building a mypy program.
 
     It is used to coordinate parsing, import processing, semantic
@@ -512,7 +510,9 @@ def __init__(self, data_dir: str,
                  stdout: TextIO,
                  stderr: TextIO,
                  ) -> None:
-        super().__init__(stdout, stderr)
+        self.stats = {}  # type: Dict[str, Any]  # Values are ints or floats
+        self.stdout = stdout
+        self.stderr = stderr
         self.start_time = time.time()
         self.data_dir = data_dir
         self.errors = errors
@@ -748,6 +748,41 @@ def report_file(self,
         if self.reports is not None and self.source_set.is_source(file):
             self.reports.file(file, type_map, options)
 
+    def verbosity(self) -> int:
+        return self.options.verbosity
+
+    def log(self, *message: str) -> None:
+        if self.verbosity() >= 1:
+            if message:
+                print('LOG: ', *message, file=self.stderr)
+            else:
+                print(file=self.stderr)
+            self.stderr.flush()
+
+    def log_fine_grained(self, *message: str) -> None:
+        import mypy.build
+        if self.verbosity() >= 1:
+            self.log('fine-grained:', *message)
+        elif mypy.build.DEBUG_FINE_GRAINED:
+            # Output log in a simplified format that is quick to browse.
+            if message:
+                print(*message, file=self.stderr)
+            else:
+                print(file=self.stderr)
+            self.stderr.flush()
+
+    def trace(self, *message: str) -> None:
+        if self.verbosity() >= 2:
+            print('TRACE:', *message, file=self.stderr)
+            self.stderr.flush()
+
+    def add_stats(self, **kwds: Any) -> None:
+        for key, value in kwds.items():
+            if key in self.stats:
+                self.stats[key] += value
+            else:
+                self.stats[key] = value
+
     def stats_summary(self) -> Mapping[str, object]:
         return self.stats
 

mypy/dmypy/__main__.py

@@ -0,0 +1,4 @@
+from mypy.dmypy.client import console_entry
+
+if __name__ == '__main__':
+    console_entry()

mypy/dmypy.py → mypy/dmypy/client.py

@@ -31,7 +31,8 @@ def __init__(self, prog: str) -> None:
         super().__init__(prog=prog, max_help_position=30)
 
 
-parser = argparse.ArgumentParser(description="Client for mypy daemon mode",
+parser = argparse.ArgumentParser(prog='dmypy',
+                                 description="Client for mypy daemon mode",
                                  fromfile_prefix_chars='@')
 parser.set_defaults(action=None)
 parser.add_argument('--status-file', default=DEFAULT_STATUS_FILE,
@@ -460,7 +461,7 @@ def request(status_file: str, command: str, *, timeout: Optional[int] = None,
     """
     response = {}  # type: Dict[str, str]
     args = dict(kwds)
-    args.update(command=command)
+    args['command'] = command
     bdata = json.dumps(args).encode('utf8')
     _, name = get_status(status_file)
     try:
@@ -537,7 +538,3 @@ def is_running(status_file: str) -> bool:
 # Run main().
 def console_entry() -> None:
     main(sys.argv[1:])
-
-
-if __name__ == '__main__':
-    console_entry()

mypy/dmypy_server.py

@@ -519,15 +519,11 @@ def _find_changed(self, sources: List[BuildSource],
     def cmd_suggest(self,
                     function: str,
                     callsites: bool,
-                    # We'd like to just use **kwargs here and save some duplication but
-                    # mypyc doesn't support it yet...
-                    json: bool,
-                    no_errors: bool,
-                    no_any: bool) -> Dict[str, object]:
+                    **kwargs: bool) -> Dict[str, object]:
         """Suggest a signature for a function."""
         if not self.fine_grained_manager:
             return {'error': "Command 'suggest' is only valid after a 'check' command"}
-        engine = SuggestionEngine(self.fine_grained_manager, json, no_errors, no_any)
+        engine = SuggestionEngine(self.fine_grained_manager, **kwargs)
         try:
             if callsites:
                 out = engine.suggest_callsites(function)

mypy/main.py

@@ -23,6 +23,7 @@
 from mypy.errors import CompileError
 from mypy.options import Options, BuildType, PER_MODULE_OPTIONS
 from mypy.config_parser import parse_version, parse_config_file
+from mypy.split_namespace import SplitNamespace
 
 from mypy.version import __version__
 
@@ -123,28 +124,6 @@ def flush_errors(new_messages: List[str], serious: bool) -> None:
         sys.exit(code)
 
 
-class SplitNamespace(argparse.Namespace):
-    def __init__(self, standard_namespace: object, alt_namespace: object, alt_prefix: str) -> None:
-        self.__dict__['_standard_namespace'] = standard_namespace
-        self.__dict__['_alt_namespace'] = alt_namespace
-        self.__dict__['_alt_prefix'] = alt_prefix
-
-    def _get(self) -> Tuple[Any, Any]:
-        return (self._standard_namespace, self._alt_namespace)
-
-    def __setattr__(self, name: str, value: Any) -> None:
-        if name.startswith(self._alt_prefix):
-            setattr(self._alt_namespace, name[len(self._alt_prefix):], value)
-        else:
-            setattr(self._standard_namespace, name, value)
-
-    def __getattr__(self, name: str) -> Any:
-        if name.startswith(self._alt_prefix):
-            return getattr(self._alt_namespace, name[len(self._alt_prefix):])
-        else:
-            return getattr(self._standard_namespace, name)
-
-
 # Make the help output a little less jarring.
 class AugmentedHelpFormatter(argparse.RawDescriptionHelpFormatter):
     def __init__(self, prog: str) -> None:
@@ -799,8 +778,11 @@ def add_invertible_flag(flag: str,
     else:
         try:
             targets = create_source_list(special_opts.files, options, fscache)
-        except InvalidSourceList as e:
-            fail(str(e), stderr)
+        # Variable named e2 instead of e to work around mypyc bug #620
+        # which causes issues when using the same variable to catch
+        # exceptions of different types.
+        except InvalidSourceList as e2:
+            fail(str(e2), stderr)
         return targets, options
 
 

mypy/split_namespace.py

@@ -0,0 +1,34 @@
+"""Split namespace for argparse to allow separating options by prefix.
+
+We use this to direct some options to an Options object and some to a
+regular namespace.
+"""
+
+# In its own file largely because mypyc doesn't support its use of
+# __getattr__/__setattr__ and has some issues with __dict__
+
+import argparse
+
+from typing import Tuple, Any
+
+
+class SplitNamespace(argparse.Namespace):
+    def __init__(self, standard_namespace: object, alt_namespace: object, alt_prefix: str) -> None:
+        self.__dict__['_standard_namespace'] = standard_namespace
+        self.__dict__['_alt_namespace'] = alt_namespace
+        self.__dict__['_alt_prefix'] = alt_prefix
+
+    def _get(self) -> Tuple[Any, Any]:
+        return (self._standard_namespace, self._alt_namespace)
+
+    def __setattr__(self, name: str, value: Any) -> None:
+        if name.startswith(self._alt_prefix):
+            setattr(self._alt_namespace, name[len(self._alt_prefix):], value)
+        else:
+            setattr(self._standard_namespace, name, value)
+
+    def __getattr__(self, name: str) -> Any:
+        if name.startswith(self._alt_prefix):
+            return getattr(self._alt_namespace, name[len(self._alt_prefix):])
+        else:
+            return getattr(self._standard_namespace, name)

setup.py

@@ -86,20 +86,22 @@ def run(self):
 
 if USE_MYPYC:
     MYPYC_BLACKLIST = (
-        # Designed to collect things that can't be compiled
-        'mypyc_hacks.py',
-        'interpreted_plugin.py',
-
-        # Can't be compiled because they need to be runnable as scripts
+        # Need to be runnable as scripts
         '__main__.py',
         'sitepkgs.py',
+        'dmypy/__main__.py',
+
+        # Needs to be interpreted to provide a hook to interpreted plugins
+        'interpreted_plugin.py',
+
+        # Uses __getattr__/__setattr__
+        'split_namespace.py',
 
-        # Can't be compiled because something goes wrong
+        # Lies to mypy about code reachability
         'bogus_type.py',
-        'dmypy.py',
-        'gclogger.py',
-        'main.py',
-        'memprofile.py',
+
+        # We don't populate __file__ properly at the top level or something?
+        # Also I think there would be problems with how we generate version.py.
         'version.py',
     )
 
@@ -169,7 +171,7 @@ def run(self):
       package_data={'mypy': package_data},
       entry_points={'console_scripts': ['mypy=mypy.__main__:console_entry',
                                         'stubgen=mypy.stubgen:main',
-                                        'dmypy=mypy.dmypy:console_entry',
+                                        'dmypy=mypy.dmypy.client:console_entry',
                                         ]},
       classifiers=classifiers,
       cmdclass=cmdclass,