Make testfinegrained use dmypy_server (#4699)

Author: msullivan

mypy/dmypy.py

@@ -147,8 +147,8 @@ def do_restart(args: argparse.Namespace) -> None:
 def start_server(args: argparse.Namespace) -> None:
     """Start the server from command arguments and wait for it."""
     # Lazy import so this import doesn't slow down other commands.
-    from mypy.dmypy_server import daemonize, Server
-    if daemonize(Server(args.flags).serve, args.log_file) != 0:
+    from mypy.dmypy_server import daemonize, Server, process_start_options
+    if daemonize(Server(process_start_options(args.flags)).serve, args.log_file) != 0:
         sys.exit(1)
     wait_for_server()
 
@@ -283,8 +283,8 @@ def do_hang(args: argparse.Namespace) -> None:
 def do_daemon(args: argparse.Namespace) -> None:
     """Serve requests in the foreground."""
     # Lazy import so this import doesn't slow down other commands.
-    from mypy.dmypy_server import Server
-    Server(args.flags).serve()
+    from mypy.dmypy_server import Server, process_start_options
+    Server(process_start_options(args.flags)).serve()
 
 
 @action(help_parser)

mypy/dmypy_server.py

@@ -20,11 +20,12 @@
 import mypy.build
 import mypy.errors
 import mypy.main
-import mypy.server.update
+from mypy.server.update import FineGrainedBuildManager
 from mypy.dmypy_util import STATUS_FILE, receive
 from mypy.gclogger import GcLogger
 from mypy.fscache import FileSystemCache
 from mypy.fswatcher import FileSystemWatcher, FileData
+from mypy.options import Options
 
 
 def daemonize(func: Callable[[], None], log_file: Optional[str] = None) -> int:
@@ -78,33 +79,44 @@ def daemonize(func: Callable[[], None], log_file: Optional[str] = None) -> int:
 SOCKET_NAME = 'dmypy.sock'  # In current directory.
 
 
+def process_start_options(flags: List[str]) -> Options:
+    import mypy.main
+    sources, options = mypy.main.process_options(['-i'] + flags,
+                                                 require_targets=False,
+                                                 server_options=True)
+    if sources:
+        sys.exit("dmypy: start/restart does not accept sources")
+    if options.report_dirs:
+        sys.exit("dmypy: start/restart cannot generate reports")
+    if options.junit_xml:
+        sys.exit("dmypy: start/restart does not support --junit-xml; "
+                 "pass it to check/recheck instead")
+    if not options.incremental:
+        sys.exit("dmypy: start/restart should not disable incremental mode")
+    if options.quick_and_dirty:
+        sys.exit("dmypy: start/restart should not specify quick_and_dirty mode")
+    if options.use_fine_grained_cache and not options.fine_grained_incremental:
+        sys.exit("dmypy: fine-grained cache can only be used in experimental mode")
+    # Our file change tracking can't yet handle changes to files that aren't
+    # specified in the sources list.
+    if options.follow_imports not in ('skip', 'error'):
+        sys.exit("dmypy: follow-imports must be 'skip' or 'error'")
+    return options
+
+
 class Server:
 
     # NOTE: the instance is constructed in the parent process but
     # serve() is called in the grandchild (by daemonize()).
 
-    def __init__(self, flags: List[str]) -> None:
+    def __init__(self, options: Options, alt_lib_path: Optional[str] = None) -> None:
         """Initialize the server with the desired mypy flags."""
         self.saved_cache = {}  # type: mypy.build.SavedCache
-        self.fine_grained_initialized = False
-        sources, options = mypy.main.process_options(['-i'] + flags,
-                                                     require_targets=False,
-                                                     server_options=True)
         self.fine_grained = options.fine_grained_incremental
-        if sources:
-            sys.exit("dmypy: start/restart does not accept sources")
-        if options.report_dirs:
-            sys.exit("dmypy: start/restart cannot generate reports")
-        if options.junit_xml:
-            sys.exit("dmypy: start/restart does not support --junit-xml; "
-                     "pass it to check/recheck instead")
-        if not options.incremental:
-            sys.exit("dmypy: start/restart should not disable incremental mode")
-        if options.quick_and_dirty:
-            sys.exit("dmypy: start/restart should not specify quick_and_dirty mode")
-        if options.use_fine_grained_cache and not options.fine_grained_incremental:
-            sys.exit("dmypy: fine-grained cache can only be used in experimental mode")
         self.options = options
+        self.alt_lib_path = alt_lib_path
+        self.fine_grained_manager = None  # type: Optional[FineGrainedBuildManager]
+
         if os.path.isfile(STATUS_FILE):
             os.unlink(STATUS_FILE)
         if self.fine_grained:
@@ -214,15 +226,13 @@ def cmd_recheck(self) -> Dict[str, object]:
     # Needed by tests.
     last_manager = None  # type: Optional[mypy.build.BuildManager]
 
-    def check(self, sources: List[mypy.build.BuildSource],
-              alt_lib_path: Optional[str] = None) -> Dict[str, Any]:
+    def check(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
         if self.fine_grained:
             return self.check_fine_grained(sources)
         else:
-            return self.check_default(sources, alt_lib_path)
+            return self.check_default(sources)
 
-    def check_default(self, sources: List[mypy.build.BuildSource],
-                      alt_lib_path: Optional[str] = None) -> Dict[str, Any]:
+    def check_default(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
         """Check using the default (per-file) incremental mode."""
         self.last_manager = None
         blockers = False
@@ -231,7 +241,7 @@ def check_default(self, sources: List[mypy.build.BuildSource],
                 # saved_cache is mutated in place.
                 res = mypy.build.build(sources, self.options,
                                        saved_cache=self.saved_cache,
-                                       alt_lib_path=alt_lib_path)
+                                       alt_lib_path=self.alt_lib_path)
                 msgs = res.errors
                 self.last_manager = res.manager  # type: Optional[mypy.build.BuildManager]
             except mypy.errors.CompileError as err:
@@ -254,7 +264,7 @@ def check_default(self, sources: List[mypy.build.BuildSource],
 
     def check_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
         """Check using fine-grained incremental mode."""
-        if not self.fine_grained_initialized:
+        if not self.fine_grained_manager:
             return self.initialize_fine_grained(sources)
         else:
             return self.fine_grained_increment(sources)
@@ -267,9 +277,9 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict
             # Stores the initial state of sources as a side effect.
             self.fswatcher.find_changed()
         try:
-            # TODO: alt_lib_path
             result = mypy.build.build(sources=sources,
-                                      options=self.options)
+                                      options=self.options,
+                                      alt_lib_path=self.alt_lib_path)
         except mypy.errors.CompileError as e:
             output = ''.join(s + '\n' for s in e.messages)
             if e.use_stdout:
@@ -280,8 +290,7 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict
         messages = result.errors
         manager = result.manager
         graph = result.graph
-        self.fine_grained_manager = mypy.server.update.FineGrainedBuildManager(manager, graph)
-        self.fine_grained_initialized = True
+        self.fine_grained_manager = FineGrainedBuildManager(manager, graph)
         self.previous_sources = sources
         self.fscache.flush()
 
@@ -310,6 +319,8 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict
         return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status}
 
     def fine_grained_increment(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
+        assert self.fine_grained_manager is not None
+
         t0 = time.time()
         self.update_sources(sources)
         changed = self.find_changed(sources)

mypy/test/helpers.py

@@ -3,6 +3,7 @@
 import subprocess
 import sys
 import time
+import shutil
 
 from typing import List, Dict, Tuple, Callable, Any, Optional
 
@@ -356,3 +357,22 @@ def run_command(cmdline: List[str], *, env: Optional[Dict[str, str]] = None,
         out = err = b''
         process.kill()
     return process.returncode, split_lines(out, err)
+
+
+def copy_and_fudge_mtime(source_path: str, target_path: str) -> None:
+    # In some systems, mtime has a resolution of 1 second which can
+    # cause annoying-to-debug issues when a file has the same size
+    # after a change. We manually set the mtime to circumvent this.
+    # Note that we increment the old file's mtime, which guarentees a
+    # different value, rather than incrementing the mtime after the
+    # copy, which could leave the mtime unchanged if the old file had
+    # a similarly fudged mtime.
+    new_time = None
+    if os.path.isfile(target_path):
+        new_time = os.stat(target_path).st_mtime + 1
+
+    # Use retries to work around potential flakiness on Windows (AppVeyor).
+    retry_on_error(lambda: shutil.copy(source_path, target_path))
+
+    if new_time:
+        os.utime(target_path, times=(new_time, new_time))

mypy/test/testcheck.py

@@ -12,7 +12,8 @@
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import (
     assert_string_arrays_equal, normalize_error_messages,
-    retry_on_error, update_testcase_output, parse_options
+    retry_on_error, update_testcase_output, parse_options,
+    copy_and_fudge_mtime
 )
 from mypy.errors import CompileError
 from mypy.options import Options
@@ -132,14 +133,7 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int = 0)
                         if file.endswith('.' + str(incremental_step)):
                             full = os.path.join(dn, file)
                             target = full[:-2]
-                            # Use retries to work around potential flakiness on Windows (AppVeyor).
-                            retry_on_error(lambda: shutil.copy(full, target))
-
-                            # In some systems, mtime has a resolution of 1 second which can cause
-                            # annoying-to-debug issues when a file has the same size after a
-                            # change. We manually set the mtime to circumvent this.
-                            new_time = os.stat(target).st_mtime + 1
-                            os.utime(target, times=(new_time, new_time))
+                            copy_and_fudge_mtime(full, target)
                 # Delete files scheduled to be deleted in [delete <path>.num] sections.
                 for path in testcase.deleted_paths.get(incremental_step, set()):
                     # Use retries to work around potential flakiness on Windows (AppVeyor).

mypy/test/testdmypy.py

@@ -15,6 +15,7 @@
 from mypy.test.helpers import (
     assert_string_arrays_equal, normalize_error_messages,
     retry_on_error, testcase_pyversion, update_testcase_output,
+    copy_and_fudge_mtime,
 )
 from mypy.options import Options
 
@@ -91,14 +92,7 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int) ->
                     if file.endswith('.' + str(incremental_step)):
                         full = os.path.join(dn, file)
                         target = full[:-2]
-                        # Use retries to work around potential flakiness on Windows (AppVeyor).
-                        retry_on_error(lambda: shutil.copy(full, target))
-
-                        # In some systems, mtime has a resolution of 1 second which can cause
-                        # annoying-to-debug issues when a file has the same size after a
-                        # change. We manually set the mtime to circumvent this.
-                        new_time = os.stat(target).st_mtime + 1
-                        os.utime(target, times=(new_time, new_time))
+                        copy_and_fudge_mtime(full, target)
             # Delete files scheduled to be deleted in [delete <path>.num] sections.
             for path in testcase.deleted_paths.get(incremental_step, set()):
                 # Use retries to work around potential flakiness on Windows (AppVeyor).
@@ -117,20 +111,16 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int) ->
         # Parse options after moving files (in case mypy.ini is being moved).
         options = self.parse_options(original_program_text, testcase, incremental_step)
         if incremental_step == 1:
-            server_options = []  # type: List[str]
             if 'fine-grained' in testcase.file:
-                server_options.append('--experimental')
                 options.fine_grained_incremental = True
-                options.local_partial_types = True
-            self.server = dmypy_server.Server(server_options)  # TODO: Fix ugly API
-            self.server.options = options
+            self.server = dmypy_server.Server(options, alt_lib_path=test_temp_dir)
 
         assert self.server is not None  # Set in step 1 and survives into next steps
         sources = []
         for module_name, program_path, program_text in module_data:
             # Always set to none so we're forced to reread the module in incremental mode
             sources.append(build.BuildSource(program_path, module_name, None))
-        response = self.server.check(sources, alt_lib_path=test_temp_dir)
+        response = self.server.check(sources)
         a = (response['out'] or response['err']).splitlines()
         a = normalize_error_messages(a)