From e19c06684ab3cca356b19508bf8d3d63e1478de7 Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Sun, 16 Jun 2019 18:02:45 -0700
Subject: [PATCH 01/15] Use qualified import and remove unused import

It's much easier to understand pytype.io than io, which was shadowing the builtin!
---
 tests/pytype_test.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index bb4793a20078..fd1c69ee6351 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -11,16 +11,15 @@
 """
 
 import argparse
-import collections
 import itertools
 import os
-from pytype import config
-from pytype import io
 import re
 import subprocess
 import sys
 import traceback
 
+import pytype
+
 parser = argparse.ArgumentParser(description='Pytype/typeshed tests.')
 parser.add_argument('-n', '--dry-run', action='store_true', default=False,
                     help='Don\'t actually run tests')
@@ -84,7 +83,7 @@ def run_pytype(args, dry_run, typeshed_location):
     old_typeshed_home = os.environ.get(TYPESHED_HOME, UNSET)
     os.environ[TYPESHED_HOME] = typeshed_location
     try:
-        io.parse_pyi(config.Options(args))
+        pytype.io.parse_pyi(pytype.config.Options(args))
     except Exception:
         stderr = traceback.format_exc()
     else:

From afcce4813153fd36829f8c697159ccda945700c4 Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Sun, 16 Jun 2019 18:03:24 -0700
Subject: [PATCH 02/15] Apply Black

---
 tests/pytype_test.py | 94 ++++++++++++++++++--------------------------
 1 file changed, 38 insertions(+), 56 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index fd1c69ee6351..c470e7da9a44 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -20,26 +20,21 @@
 
 import pytype
 
-parser = argparse.ArgumentParser(description='Pytype/typeshed tests.')
-parser.add_argument('-n', '--dry-run', action='store_true', default=False,
-                    help='Don\'t actually run tests')
+parser = argparse.ArgumentParser(description="Pytype/typeshed tests.")
+parser.add_argument("-n", "--dry-run", action="store_true", default=False, help="Don't actually run tests")
 # Default to '' so that symlinking typeshed subdirs in cwd will work.
-parser.add_argument('--typeshed-location', type=str, default='',
-                    help='Path to typeshed installation.')
+parser.add_argument("--typeshed-location", type=str, default="", help="Path to typeshed installation.")
 # Set to true to print a stack trace every time an exception is thrown.
-parser.add_argument('--print-stderr', action='store_true', default=False,
-                    help='Print stderr every time an error is encountered.')
+parser.add_argument("--print-stderr", action="store_true", default=False, help="Print stderr every time an error is encountered.")
 # We need to invoke python2.7 and 3.6.
-parser.add_argument('--python27-exe', type=str, default='python2.7',
-                    help='Path to a python 2.7 interpreter.')
-parser.add_argument('--python36-exe', type=str, default='python3.6',
-                    help='Path to a python 3.6 interpreter.')
+parser.add_argument("--python27-exe", type=str, default="python2.7", help="Path to a python 2.7 interpreter.")
+parser.add_argument("--python36-exe", type=str, default="python3.6", help="Path to a python 3.6 interpreter.")
 
 
-TYPESHED_SUBDIRS = ['stdlib', 'third_party']
+TYPESHED_SUBDIRS = ["stdlib", "third_party"]
 
 
-TYPESHED_HOME = 'TYPESHED_HOME'
+TYPESHED_HOME = "TYPESHED_HOME"
 UNSET = object()  # marker for tracking the TYPESHED_HOME environment variable
 
 
@@ -52,7 +47,7 @@ def main():
 class PathMatcher(object):
     def __init__(self, patterns):
         if patterns:
-            self.matcher = re.compile('(%s)$' % '|'.join(patterns))
+            self.matcher = re.compile("(%s)$" % "|".join(patterns))
         else:
             self.matcher = None
 
@@ -63,8 +58,8 @@ def search(self, path):
 
 
 def load_blacklist(typeshed_location):
-    filename = os.path.join(typeshed_location, 'tests', 'pytype_blacklist.txt')
-    skip_re = re.compile(r'^\s*([^\s#]+)\s*(?:#.*)?$')
+    filename = os.path.join(typeshed_location, "tests", "pytype_blacklist.txt")
+    skip_re = re.compile(r"^\s*([^\s#]+)\s*(?:#.*)?$")
     skip = []
 
     with open(filename) as f:
@@ -109,23 +104,20 @@ def _get_relative(filename):
 
 def _get_module_name(filename):
     """Converts a filename {subdir}/m.n/module/foo to module.foo."""
-    return '.'.join(_get_relative(filename).split(os.path.sep)[2:]).replace(
-        '.pyi', '').replace('.__init__', '')
+    return ".".join(_get_relative(filename).split(os.path.sep)[2:]).replace(".pyi", "").replace(".__init__", "")
 
 
 def can_run(path, exe, *args):
     exe = os.path.join(path, exe)
     try:
-        subprocess.Popen(
-            [exe] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE
-        ).communicate()
+        subprocess.Popen([exe] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
         return True
     except OSError:
         return False
 
 
 def _is_version(path, version):
-    return any('%s/%s' % (d, version) in path for d in TYPESHED_SUBDIRS)
+    return any("%s/%s" % (d, version) in path for d in TYPESHED_SUBDIRS)
 
 
 def pytype_test(args):
@@ -135,17 +127,15 @@ def pytype_test(args):
 
     for p in paths:
         if not os.path.isdir(p):
-            print('Cannot find typeshed subdir at %s '
-                  '(specify parent dir via --typeshed-location)' % p)
+            print("Cannot find typeshed subdir at %s " "(specify parent dir via --typeshed-location)" % p)
             return 1
 
-    for python_version_str in ('27', '36'):
-        dest = 'python%s_exe' % python_version_str
-        version = '.'.join(list(python_version_str))
-        arg = '--python%s-exe' % python_version_str
-        if not can_run('', getattr(args, dest), '--version'):
-            print('Cannot run Python {version}. (point to a valid executable '
-                  'via {arg})'.format(version=version, arg=arg))
+    for python_version_str in ("27", "36"):
+        dest = "python%s_exe" % python_version_str
+        version = ".".join(list(python_version_str))
+        arg = "--python%s-exe" % python_version_str
+        if not can_run("", getattr(args, dest), "--version"):
+            print("Cannot run Python {version}. (point to a valid executable " "via {arg})".format(version=version, arg=arg))
             return 1
 
     skipped = PathMatcher(load_blacklist(typeshed_location))
@@ -154,40 +144,32 @@ def pytype_test(args):
 
     def _parse(filename, major_version):
         if major_version == 3:
-            version = '3.6'
+            version = "3.6"
             exe = args.python36_exe
         else:
-            version = '2.7'
+            version = "2.7"
             exe = args.python27_exe
-        options = [
-            '--module-name=%s' % _get_module_name(filename),
-            '--parse-pyi',
-            '-V %s' % version,
-            '--python_exe=%s' % exe,
-        ]
-        return run_pytype(options + [filename],
-                          dry_run=args.dry_run,
-                          typeshed_location=typeshed_location)
-
-    for root, _, filenames in itertools.chain.from_iterable(
-            os.walk(p) for p in paths):
-        for f in sorted(f for f in filenames if f.endswith('.pyi')):
+        options = ["--module-name=%s" % _get_module_name(filename), "--parse-pyi", "-V %s" % version, "--python_exe=%s" % exe]
+        return run_pytype(options + [filename], dry_run=args.dry_run, typeshed_location=typeshed_location)
+
+    for root, _, filenames in itertools.chain.from_iterable(os.walk(p) for p in paths):
+        for f in sorted(f for f in filenames if f.endswith(".pyi")):
             f = os.path.join(root, f)
             rel = _get_relative(f)
             if not skipped.search(rel):
-                if _is_version(f, '2and3'):
+                if _is_version(f, "2and3"):
                     files.append((f, 2))
                     files.append((f, 3))
-                elif _is_version(f, '2'):
+                elif _is_version(f, "2"):
                     files.append((f, 2))
-                elif _is_version(f, '3'):
+                elif _is_version(f, "3"):
                     files.append((f, 3))
                 else:
-                    print('Unrecognized path: %s' % f)
+                    print("Unrecognized path: %s" % f)
 
     errors = 0
     total_tests = len(files)
-    print('Testing files with pytype...')
+    print("Testing files with pytype...")
     for i, (f, version) in enumerate(files):
         stderr = _parse(f, version)
         if stderr:
@@ -196,17 +178,17 @@ def _parse(filename, major_version):
             errors += 1
             # We strip off the stack trace and just leave the last line with the
             # actual error; to see the stack traces use --print_stderr.
-            bad.append((_get_relative(f), stderr.rstrip().rsplit('\n', 1)[-1]))
+            bad.append((_get_relative(f), stderr.rstrip().rsplit("\n", 1)[-1]))
 
         runs = i + 1
         if runs % 25 == 0:
-            print('  %3d/%d with %3d errors' % (runs, total_tests, errors))
+            print("  %3d/%d with %3d errors" % (runs, total_tests, errors))
 
-    print('Ran pytype with %d pyis, got %d errors.' % (total_tests, errors))
+    print("Ran pytype with %d pyis, got %d errors." % (total_tests, errors))
     for f, err in bad:
-        print('%s: %s' % (f, err))
+        print("%s: %s" % (f, err))
     return int(bool(errors))
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

From d785754eba57bc64fbcb7869b74d2e67b98fc0d9 Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Sun, 16 Jun 2019 18:05:50 -0700
Subject: [PATCH 03/15] Use Python 3

* Set shebang
* Classes are always new type
---
 tests/pytype_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index c470e7da9a44..20d09c924887 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-r"""Test runner for typeshed.
+#!/usr/bin/env python3
+"""Test runner for typeshed.
 
 Depends on pytype being installed.
 
@@ -44,7 +44,7 @@ def main():
     sys.exit(code)
 
 
-class PathMatcher(object):
+class PathMatcher:
     def __init__(self, patterns):
         if patterns:
             self.matcher = re.compile("(%s)$" % "|".join(patterns))

From 8730bc35f8335362e89406bcfbaace7208861821 Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Sun, 16 Jun 2019 18:07:37 -0700
Subject: [PATCH 04/15] Make function create_parser()

Results in less implementation details being exposed at the top level of the module.
---
 tests/pytype_test.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index 20d09c924887..b357ff601f15 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -20,17 +20,6 @@
 
 import pytype
 
-parser = argparse.ArgumentParser(description="Pytype/typeshed tests.")
-parser.add_argument("-n", "--dry-run", action="store_true", default=False, help="Don't actually run tests")
-# Default to '' so that symlinking typeshed subdirs in cwd will work.
-parser.add_argument("--typeshed-location", type=str, default="", help="Path to typeshed installation.")
-# Set to true to print a stack trace every time an exception is thrown.
-parser.add_argument("--print-stderr", action="store_true", default=False, help="Print stderr every time an error is encountered.")
-# We need to invoke python2.7 and 3.6.
-parser.add_argument("--python27-exe", type=str, default="python2.7", help="Path to a python 2.7 interpreter.")
-parser.add_argument("--python36-exe", type=str, default="python3.6", help="Path to a python 3.6 interpreter.")
-
-
 TYPESHED_SUBDIRS = ["stdlib", "third_party"]
 
 
@@ -39,11 +28,26 @@
 
 
 def main():
-    args = parser.parse_args()
+    args = create_parser().parse_args()
     code = pytype_test(args)
     sys.exit(code)
 
 
+def create_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Pytype/typeshed tests.")
+    parser.add_argument("-n", "--dry-run", action="store_true", default=False, help="Don't actually run tests")
+    # Default to '' so that symlinking typeshed subdirs in cwd will work.
+    parser.add_argument("--typeshed-location", type=str, default="", help="Path to typeshed installation.")
+    # Set to true to print a stack trace every time an exception is thrown.
+    parser.add_argument(
+        "--print-stderr", action="store_true", default=False, help="Print stderr every time an error is encountered."
+    )
+    # We need to invoke python2.7 and 3.6.
+    parser.add_argument("--python27-exe", type=str, default="python2.7", help="Path to a python 2.7 interpreter.")
+    parser.add_argument("--python36-exe", type=str, default="python3.6", help="Path to a python 3.6 interpreter.")
+    return parser
+
+
 class PathMatcher:
     def __init__(self, patterns):
         if patterns:

From 1641bb70955ecb226bd570015df9daf9ffdc7ba4 Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Sun, 16 Jun 2019 18:15:57 -0700
Subject: [PATCH 05/15] Use new style string interpolation

---
 tests/pytype_test.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index b357ff601f15..a6d22487f3ff 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -51,7 +51,7 @@ def create_parser() -> argparse.ArgumentParser:
 class PathMatcher:
     def __init__(self, patterns):
         if patterns:
-            self.matcher = re.compile("(%s)$" % "|".join(patterns))
+            self.matcher = re.compile(r"({})$".format("|".join(patterns)))
         else:
             self.matcher = None
 
@@ -121,7 +121,7 @@ def can_run(path, exe, *args):
 
 
 def _is_version(path, version):
-    return any("%s/%s" % (d, version) in path for d in TYPESHED_SUBDIRS)
+    return any("{}/{}".format(d, version) in path for d in TYPESHED_SUBDIRS)
 
 
 def pytype_test(args):
@@ -131,15 +131,15 @@ def pytype_test(args):
 
     for p in paths:
         if not os.path.isdir(p):
-            print("Cannot find typeshed subdir at %s " "(specify parent dir via --typeshed-location)" % p)
+            print("Cannot find typeshed subdir at {} (specify parent dir via --typeshed-location)".format(p))
             return 1
 
     for python_version_str in ("27", "36"):
-        dest = "python%s_exe" % python_version_str
+        dest = "python{}_exe".format(python_version_str)
         version = ".".join(list(python_version_str))
-        arg = "--python%s-exe" % python_version_str
+        arg = "--python{}-exe".format(python_version_str)
         if not can_run("", getattr(args, dest), "--version"):
-            print("Cannot run Python {version}. (point to a valid executable " "via {arg})".format(version=version, arg=arg))
+            print("Cannot run Python {version}. (point to a valid executable via {arg})".format(version=version, arg=arg))
             return 1
 
     skipped = PathMatcher(load_blacklist(typeshed_location))
@@ -153,7 +153,7 @@ def _parse(filename, major_version):
         else:
             version = "2.7"
             exe = args.python27_exe
-        options = ["--module-name=%s" % _get_module_name(filename), "--parse-pyi", "-V %s" % version, "--python_exe=%s" % exe]
+        options = ["--module-name={}".format(_get_module_name(filename)), "--parse-pyi", "-V {}".format(version), "--python_exe={}".format(exe)]
         return run_pytype(options + [filename], dry_run=args.dry_run, typeshed_location=typeshed_location)
 
     for root, _, filenames in itertools.chain.from_iterable(os.walk(p) for p in paths):
@@ -169,7 +169,7 @@ def _parse(filename, major_version):
                 elif _is_version(f, "3"):
                     files.append((f, 3))
                 else:
-                    print("Unrecognized path: %s" % f)
+                    print("Unrecognized path: {}".format(f))
 
     errors = 0
     total_tests = len(files)
@@ -186,11 +186,11 @@ def _parse(filename, major_version):
 
         runs = i + 1
         if runs % 25 == 0:
-            print("  %3d/%d with %3d errors" % (runs, total_tests, errors))
+            print("  {:3d}/{:d} with {:3d} errors".format(runs, total_tests, errors))
 
-    print("Ran pytype with %d pyis, got %d errors." % (total_tests, errors))
+    print("Ran pytype with {:d} pyis, got {:d} errors.".format(total_tests, errors))
     for f, err in bad:
-        print("%s: %s" % (f, err))
+        print("{}: {}".format(f, err))
     return int(bool(errors))
 
 

From 9b2346c8837ff0eba91f0fbbee168487d6d3833c Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Sun, 16 Jun 2019 18:19:40 -0700
Subject: [PATCH 06/15] Use subprocess.run()

---
 tests/pytype_test.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index a6d22487f3ff..a27c9593f98a 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -114,10 +114,11 @@ def _get_module_name(filename):
 def can_run(path, exe, *args):
     exe = os.path.join(path, exe)
     try:
-        subprocess.Popen([exe] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
-        return True
+        subprocess.run([exe] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     except OSError:
         return False
+    else:
+        return True
 
 
 def _is_version(path, version):

From 68a221fcfc0d424a274011b3640f7b170691d610 Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Sun, 16 Jun 2019 18:40:03 -0700
Subject: [PATCH 07/15] Add initial type hints

---
 tests/pytype_test.py | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index a27c9593f98a..1c193da9c4cd 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -17,6 +17,7 @@
 import subprocess
 import sys
 import traceback
+from typing import List, Optional, Sequence, Match
 
 import pytype
 
@@ -27,7 +28,7 @@
 UNSET = object()  # marker for tracking the TYPESHED_HOME environment variable
 
 
-def main():
+def main() -> None:
     args = create_parser().parse_args()
     code = pytype_test(args)
     sys.exit(code)
@@ -49,19 +50,16 @@ def create_parser() -> argparse.ArgumentParser:
 
 
 class PathMatcher:
-    def __init__(self, patterns):
-        if patterns:
-            self.matcher = re.compile(r"({})$".format("|".join(patterns)))
-        else:
-            self.matcher = None
+    def __init__(self, patterns: Sequence[str]) -> None:
+        self.matcher = re.compile(r"({})$".format("|".join(patterns))) if patterns else None
 
-    def search(self, path):
+    def search(self, path: str) -> Optional[Match[str]]:
         if not self.matcher:
-            return False
+            return None
         return self.matcher.search(path)
 
 
-def load_blacklist(typeshed_location):
+def load_blacklist(typeshed_location: str) -> List[str]:
     filename = os.path.join(typeshed_location, "tests", "pytype_blacklist.txt")
     skip_re = re.compile(r"^\s*([^\s#]+)\s*(?:#.*)?$")
     skip = []
@@ -75,7 +73,7 @@ def load_blacklist(typeshed_location):
     return skip
 
 
-def run_pytype(args, dry_run, typeshed_location):
+def run_pytype(args: Sequence[str], dry_run: bool, typeshed_location: str) -> Optional[str]:
     """Runs pytype, returning the stderr if any."""
     if dry_run:
         return None
@@ -94,7 +92,7 @@ def run_pytype(args, dry_run, typeshed_location):
     return stderr
 
 
-def _get_relative(filename):
+def _get_relative(filename: str) -> str:
     top = 0
     for d in TYPESHED_SUBDIRS:
         try:
@@ -106,7 +104,7 @@ def _get_relative(filename):
     return filename[top:]
 
 
-def _get_module_name(filename):
+def _get_module_name(filename: str) -> str:
     """Converts a filename {subdir}/m.n/module/foo to module.foo."""
     return ".".join(_get_relative(filename).split(os.path.sep)[2:]).replace(".pyi", "").replace(".__init__", "")
 
@@ -121,11 +119,11 @@ def can_run(path, exe, *args):
         return True
 
 
-def _is_version(path, version):
+def _is_version(path: str, version: str) -> bool:
     return any("{}/{}".format(d, version) in path for d in TYPESHED_SUBDIRS)
 
 
-def pytype_test(args):
+def pytype_test(args: argparse.Namespace) -> int:
     """Test with pytype, returning 0 for success and 1 for failure."""
     typeshed_location = args.typeshed_location or os.getcwd()
     paths = [os.path.join(typeshed_location, d) for d in TYPESHED_SUBDIRS]
@@ -147,7 +145,7 @@ def pytype_test(args):
     files = []
     bad = []
 
-    def _parse(filename, major_version):
+    def _parse(filename: str, major_version: int) -> Optional[str]:
         if major_version == 3:
             version = "3.6"
             exe = args.python36_exe

From 9e975352b7f76e56a7f04f98df7b98647c459927 Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Sun, 16 Jun 2019 18:43:53 -0700
Subject: [PATCH 08/15] Print to user tip to use --print_stderr upon failure

---
 tests/pytype_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index 1c193da9c4cd..ded7946abc29 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -179,9 +179,8 @@ def _parse(filename: str, major_version: int) -> Optional[str]:
             if args.print_stderr:
                 print(stderr)
             errors += 1
-            # We strip off the stack trace and just leave the last line with the
-            # actual error; to see the stack traces use --print_stderr.
-            bad.append((_get_relative(f), stderr.rstrip().rsplit("\n", 1)[-1]))
+            stacktrace_final_line = stderr.rstrip().rsplit("\n", 1)[-1]
+            bad.append((_get_relative(f), stacktrace_final_line))
 
         runs = i + 1
         if runs % 25 == 0:
@@ -190,6 +189,7 @@ def _parse(filename: str, major_version: int) -> Optional[str]:
     print("Ran pytype with {:d} pyis, got {:d} errors.".format(total_tests, errors))
     for f, err in bad:
         print("{}: {}".format(f, err))
+    print("\nRun again with --print-stderr to get the full stacktrace.")
     return int(bool(errors))
 
 

From 6e1d2dcf7e1ba717b7e3a4828a6584bbb46502d3 Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Sun, 16 Jun 2019 18:50:13 -0700
Subject: [PATCH 09/15] Move out dry_run logic from run_pytype

It makes that function a little bit easier to understand. We don't need to keep re-evaluating the value every time.
---
 tests/pytype_test.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index ded7946abc29..6dd8c507b099 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -73,10 +73,8 @@ def load_blacklist(typeshed_location: str) -> List[str]:
     return skip
 
 
-def run_pytype(args: Sequence[str], dry_run: bool, typeshed_location: str) -> Optional[str]:
+def run_pytype(args: Sequence[str], typeshed_location: str) -> Optional[str]:
     """Runs pytype, returning the stderr if any."""
-    if dry_run:
-        return None
     old_typeshed_home = os.environ.get(TYPESHED_HOME, UNSET)
     os.environ[TYPESHED_HOME] = typeshed_location
     try:
@@ -141,6 +139,9 @@ def pytype_test(args: argparse.Namespace) -> int:
             print("Cannot run Python {version}. (point to a valid executable via {arg})".format(version=version, arg=arg))
             return 1
 
+    if args.dry_run:
+        return 0
+
     skipped = PathMatcher(load_blacklist(typeshed_location))
     files = []
     bad = []
@@ -153,7 +154,7 @@ def _parse(filename: str, major_version: int) -> Optional[str]:
             version = "2.7"
             exe = args.python27_exe
         options = ["--module-name={}".format(_get_module_name(filename)), "--parse-pyi", "-V {}".format(version), "--python_exe={}".format(exe)]
-        return run_pytype(options + [filename], dry_run=args.dry_run, typeshed_location=typeshed_location)
+        return run_pytype(options + [filename], typeshed_location=typeshed_location)
 
     for root, _, filenames in itertools.chain.from_iterable(os.walk(p) for p in paths):
         for f in sorted(f for f in filenames if f.endswith(".pyi")):

From adafff6582e6fc4c8562ed628750394b52aacbba Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Sun, 16 Jun 2019 19:26:24 -0700
Subject: [PATCH 10/15] Extract check_python_exes_runnable() and
 check_subdirs_discoverable() so that pytype_test() is more focused

The logic belonged in main(). In the process, those two functions are refactored.
---
 tests/pytype_test.py | 78 ++++++++++++++++++++++++++------------------
 1 file changed, 47 insertions(+), 31 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index 6dd8c507b099..07eb0645cde0 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -15,7 +15,6 @@
 import os
 import re
 import subprocess
-import sys
 import traceback
 from typing import List, Optional, Sequence, Match
 
@@ -30,8 +29,19 @@
 
 def main() -> None:
     args = create_parser().parse_args()
-    code = pytype_test(args)
-    sys.exit(code)
+    typeshed_location = args.typeshed_location or os.getcwd()
+    subdir_paths = [os.path.join(typeshed_location, d) for d in TYPESHED_SUBDIRS]
+    check_subdirs_discoverable(subdir_paths)
+    check_python_exes_runnable(python27_exe_arg=args.python27_exe, python36_exe_arg=args.python36_exe)
+    if args.dry_run:
+        return
+    pytype_test(
+        typeshed_location=typeshed_location,
+        subdir_paths=subdir_paths,
+        python27_exe=args.python27_exe,
+        python36_exe=args.python36_exe,
+        print_stderr=args.print_stderr,
+    )
 
 
 def create_parser() -> argparse.ArgumentParser:
@@ -107,10 +117,9 @@ def _get_module_name(filename: str) -> str:
     return ".".join(_get_relative(filename).split(os.path.sep)[2:]).replace(".pyi", "").replace(".__init__", "")
 
 
-def can_run(path, exe, *args):
-    exe = os.path.join(path, exe)
+def can_run(exe: str, *, args: List[str]) -> bool:
     try:
-        subprocess.run([exe] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        subprocess.run([exe] + args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
     except OSError:
         return False
     else:
@@ -121,27 +130,29 @@ def _is_version(path: str, version: str) -> bool:
     return any("{}/{}".format(d, version) in path for d in TYPESHED_SUBDIRS)
 
 
-def pytype_test(args: argparse.Namespace) -> int:
-    """Test with pytype, returning 0 for success and 1 for failure."""
-    typeshed_location = args.typeshed_location or os.getcwd()
-    paths = [os.path.join(typeshed_location, d) for d in TYPESHED_SUBDIRS]
-
-    for p in paths:
+def check_subdirs_discoverable(subdir_paths: str) -> None:
+    for p in subdir_paths:
         if not os.path.isdir(p):
-            print("Cannot find typeshed subdir at {} (specify parent dir via --typeshed-location)".format(p))
-            return 1
-
-    for python_version_str in ("27", "36"):
-        dest = "python{}_exe".format(python_version_str)
-        version = ".".join(list(python_version_str))
-        arg = "--python{}-exe".format(python_version_str)
-        if not can_run("", getattr(args, dest), "--version"):
-            print("Cannot run Python {version}. (point to a valid executable via {arg})".format(version=version, arg=arg))
-            return 1
+            raise SystemExit("Cannot find typeshed subdir at {} (specify parent dir via --typeshed-location)".format(p))
 
-    if args.dry_run:
-        return 0
 
+def check_python_exes_runnable(*, python27_exe_arg: str, python36_exe_arg: str) -> None:
+    for exe, version_str in zip([python27_exe_arg, python36_exe_arg], ["27", "3.6"]):
+        if can_run(exe, args=["--version"]):
+            continue
+        formatted_version = ".".join(list(version_str))
+        script_arg = "--python{}-exe".format(version_str)
+        raise SystemExit(
+            "Cannot run Python {version}. (point to a valid executable via {arg})".format(
+                version=formatted_version, arg=script_arg
+            )
+        )
+
+
+def pytype_test(
+    *, typeshed_location: str, subdir_paths: Sequence[str], python27_exe: str, python36_exe: str, print_stderr: bool
+) -> None:
+    """Test with pytype."""
     skipped = PathMatcher(load_blacklist(typeshed_location))
     files = []
     bad = []
@@ -149,14 +160,19 @@ def pytype_test(args: argparse.Namespace) -> int:
     def _parse(filename: str, major_version: int) -> Optional[str]:
         if major_version == 3:
             version = "3.6"
-            exe = args.python36_exe
+            exe = python36_exe
         else:
             version = "2.7"
-            exe = args.python27_exe
-        options = ["--module-name={}".format(_get_module_name(filename)), "--parse-pyi", "-V {}".format(version), "--python_exe={}".format(exe)]
+            exe = python27_exe
+        options = [
+            "--module-name={}".format(_get_module_name(filename)),
+            "--parse-pyi",
+            "-V {}".format(version),
+            "--python_exe={}".format(exe),
+        ]
         return run_pytype(options + [filename], typeshed_location=typeshed_location)
 
-    for root, _, filenames in itertools.chain.from_iterable(os.walk(p) for p in paths):
+    for root, _, filenames in itertools.chain.from_iterable(os.walk(p) for p in subdir_paths):
         for f in sorted(f for f in filenames if f.endswith(".pyi")):
             f = os.path.join(root, f)
             rel = _get_relative(f)
@@ -177,7 +193,7 @@ def _parse(filename: str, major_version: int) -> Optional[str]:
     for i, (f, version) in enumerate(files):
         stderr = _parse(f, version)
         if stderr:
-            if args.print_stderr:
+            if print_stderr:
                 print(stderr)
             errors += 1
             stacktrace_final_line = stderr.rstrip().rsplit("\n", 1)[-1]
@@ -190,8 +206,8 @@ def _parse(filename: str, major_version: int) -> Optional[str]:
     print("Ran pytype with {:d} pyis, got {:d} errors.".format(total_tests, errors))
     for f, err in bad:
         print("{}: {}".format(f, err))
-    print("\nRun again with --print-stderr to get the full stacktrace.")
-    return int(bool(errors))
+    if errors:
+        raise SystemExit("\nRun again with --print-stderr to get the full stacktrace.")
 
 
 if __name__ == "__main__":

From 54234794b4c7f71523c9cdd64dcbab9bc48757e9 Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Sun, 16 Jun 2019 19:35:41 -0700
Subject: [PATCH 11/15] Extract determine_files_to_test()

---
 tests/pytype_test.py | 54 +++++++++++++++++++++++++-------------------
 1 file changed, 31 insertions(+), 23 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index 07eb0645cde0..b8104f8f1848 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -16,7 +16,7 @@
 import re
 import subprocess
 import traceback
-from typing import List, Optional, Sequence, Match
+from typing import List, Match, Optional, Sequence, Tuple
 
 import pytype
 
@@ -35,9 +35,10 @@ def main() -> None:
     check_python_exes_runnable(python27_exe_arg=args.python27_exe, python36_exe_arg=args.python36_exe)
     if args.dry_run:
         return
+    files_to_test = determine_files_to_test(typeshed_location=typeshed_location, subdir_paths=subdir_paths)
     pytype_test(
+        files_to_test=files_to_test,
         typeshed_location=typeshed_location,
-        subdir_paths=subdir_paths,
         python27_exe=args.python27_exe,
         python36_exe=args.python36_exe,
         print_stderr=args.print_stderr,
@@ -149,13 +150,34 @@ def check_python_exes_runnable(*, python27_exe_arg: str, python36_exe_arg: str)
         )
 
 
+def determine_files_to_test(*, typeshed_location: str, subdir_paths: Sequence[str]) -> List[Tuple[str, int]]:
+    """Determine all files to test, checking if it's in the blacklist and which Python versions to use.
+
+    Returns a list of pairs of the file path and Python version as an int."""
+    skipped = PathMatcher(load_blacklist(typeshed_location))
+    files = []
+    for root, _, filenames in itertools.chain.from_iterable(os.walk(p) for p in subdir_paths):
+        for f in sorted(f for f in filenames if f.endswith(".pyi")):
+            f = os.path.join(root, f)
+            rel = _get_relative(f)
+            if skipped.search(rel):
+                continue
+            if _is_version(f, "2and3"):
+                files.append((f, 2))
+                files.append((f, 3))
+            elif _is_version(f, "2"):
+                files.append((f, 2))
+            elif _is_version(f, "3"):
+                files.append((f, 3))
+            else:
+                print("Unrecognized path: {}".format(f))
+    return files
+
+
 def pytype_test(
-    *, typeshed_location: str, subdir_paths: Sequence[str], python27_exe: str, python36_exe: str, print_stderr: bool
+    *, files_to_test: Sequence[Tuple[str, int]], typeshed_location: str, python27_exe: str, python36_exe: str, print_stderr: bool
 ) -> None:
     """Test with pytype."""
-    skipped = PathMatcher(load_blacklist(typeshed_location))
-    files = []
-    bad = []
 
     def _parse(filename: str, major_version: int) -> Optional[str]:
         if major_version == 3:
@@ -172,25 +194,11 @@ def _parse(filename: str, major_version: int) -> Optional[str]:
         ]
         return run_pytype(options + [filename], typeshed_location=typeshed_location)
 
-    for root, _, filenames in itertools.chain.from_iterable(os.walk(p) for p in subdir_paths):
-        for f in sorted(f for f in filenames if f.endswith(".pyi")):
-            f = os.path.join(root, f)
-            rel = _get_relative(f)
-            if not skipped.search(rel):
-                if _is_version(f, "2and3"):
-                    files.append((f, 2))
-                    files.append((f, 3))
-                elif _is_version(f, "2"):
-                    files.append((f, 2))
-                elif _is_version(f, "3"):
-                    files.append((f, 3))
-                else:
-                    print("Unrecognized path: {}".format(f))
-
+    bad = []
     errors = 0
-    total_tests = len(files)
+    total_tests = len(files_to_test)
     print("Testing files with pytype...")
-    for i, (f, version) in enumerate(files):
+    for i, (f, version) in enumerate(files_to_test):
         stderr = _parse(f, version)
         if stderr:
             if print_stderr:

From 19d7d957ba9438411959ac639cce2157905fa89e Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Sun, 16 Jun 2019 19:47:13 -0700
Subject: [PATCH 12/15] Make run_pytype less generic

It is only ever used to do one thing. This allows us to remove the _parse() helper function.
---
 tests/pytype_test.py | 43 ++++++++++++++++++++-----------------------
 1 file changed, 20 insertions(+), 23 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index b8104f8f1848..7b99f242b4b1 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -36,7 +36,7 @@ def main() -> None:
     if args.dry_run:
         return
     files_to_test = determine_files_to_test(typeshed_location=typeshed_location, subdir_paths=subdir_paths)
-    pytype_test(
+    run_all_tests(
         files_to_test=files_to_test,
         typeshed_location=typeshed_location,
         python27_exe=args.python27_exe,
@@ -84,12 +84,21 @@ def load_blacklist(typeshed_location: str) -> List[str]:
     return skip
 
 
-def run_pytype(args: Sequence[str], typeshed_location: str) -> Optional[str]:
+def run_pytype(*, filename: str, python_version: str, python_exe: str, typeshed_location: str) -> Optional[str]:
     """Runs pytype, returning the stderr if any."""
+    options = pytype.config.Options(
+        [
+            "--module-name={}".format(_get_module_name(filename)),
+            "--parse-pyi",
+            "-V {}".format(python_version),
+            "--python_exe={}".format(python_exe),
+            filename,
+        ]
+    )
     old_typeshed_home = os.environ.get(TYPESHED_HOME, UNSET)
     os.environ[TYPESHED_HOME] = typeshed_location
     try:
-        pytype.io.parse_pyi(pytype.config.Options(args))
+        pytype.io.parse_pyi(options)
     except Exception:
         stderr = traceback.format_exc()
     else:
@@ -131,7 +140,7 @@ def _is_version(path: str, version: str) -> bool:
     return any("{}/{}".format(d, version) in path for d in TYPESHED_SUBDIRS)
 
 
-def check_subdirs_discoverable(subdir_paths: str) -> None:
+def check_subdirs_discoverable(subdir_paths: List[str]) -> None:
     for p in subdir_paths:
         if not os.path.isdir(p):
             raise SystemExit("Cannot find typeshed subdir at {} (specify parent dir via --typeshed-location)".format(p))
@@ -174,32 +183,20 @@ def determine_files_to_test(*, typeshed_location: str, subdir_paths: Sequence[st
     return files
 
 
-def pytype_test(
+def run_all_tests(
     *, files_to_test: Sequence[Tuple[str, int]], typeshed_location: str, python27_exe: str, python36_exe: str, print_stderr: bool
 ) -> None:
-    """Test with pytype."""
-
-    def _parse(filename: str, major_version: int) -> Optional[str]:
-        if major_version == 3:
-            version = "3.6"
-            exe = python36_exe
-        else:
-            version = "2.7"
-            exe = python27_exe
-        options = [
-            "--module-name={}".format(_get_module_name(filename)),
-            "--parse-pyi",
-            "-V {}".format(version),
-            "--python_exe={}".format(exe),
-        ]
-        return run_pytype(options + [filename], typeshed_location=typeshed_location)
-
     bad = []
     errors = 0
     total_tests = len(files_to_test)
     print("Testing files with pytype...")
     for i, (f, version) in enumerate(files_to_test):
-        stderr = _parse(f, version)
+        stderr = run_pytype(
+            filename=f,
+            python_version="2.7" if version == 2 else "3.6",
+            python_exe=python27_exe if version == 2 else python36_exe,
+            typeshed_location=typeshed_location,
+        )
         if stderr:
             if print_stderr:
                 print(stderr)

From e1aa9621e2415fcfbc548a3c471bd9ac52cd1e8d Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Sun, 16 Jun 2019 19:57:28 -0700
Subject: [PATCH 13/15] Fix bad imports

---
 tests/pytype_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index 7b99f242b4b1..704756ef842e 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -18,7 +18,7 @@
 import traceback
 from typing import List, Match, Optional, Sequence, Tuple
 
-import pytype
+from pytype import config as pytype_config, io as pytype_io
 
 TYPESHED_SUBDIRS = ["stdlib", "third_party"]
 
@@ -86,7 +86,7 @@ def load_blacklist(typeshed_location: str) -> List[str]:
 
 def run_pytype(*, filename: str, python_version: str, python_exe: str, typeshed_location: str) -> Optional[str]:
     """Runs pytype, returning the stderr if any."""
-    options = pytype.config.Options(
+    options = pytype_config.Options(
         [
             "--module-name={}".format(_get_module_name(filename)),
             "--parse-pyi",
@@ -98,7 +98,7 @@ def run_pytype(*, filename: str, python_version: str, python_exe: str, typeshed_
     old_typeshed_home = os.environ.get(TYPESHED_HOME, UNSET)
     os.environ[TYPESHED_HOME] = typeshed_location
     try:
-        pytype.io.parse_pyi(options)
+        pytype_io.parse_pyi(options)
     except Exception:
         stderr = traceback.format_exc()
     else:

From 1e00216e1883ae8f64ba82474127f8056f384bc8 Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Mon, 17 Jun 2019 12:00:10 -0700
Subject: [PATCH 14/15] Fix typo

---
 tests/pytype_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index 704756ef842e..655c0b530e14 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -147,7 +147,7 @@ def check_subdirs_discoverable(subdir_paths: List[str]) -> None:
 
 
 def check_python_exes_runnable(*, python27_exe_arg: str, python36_exe_arg: str) -> None:
-    for exe, version_str in zip([python27_exe_arg, python36_exe_arg], ["27", "3.6"]):
+    for exe, version_str in zip([python27_exe_arg, python36_exe_arg], ["27", "36"]):
         if can_run(exe, args=["--version"]):
             continue
         formatted_version = ".".join(list(version_str))

From 4296152de183cce28363e99ecec5b0c15c5b6fce Mon Sep 17 00:00:00 2001
From: Eric Arellano <ericarellano@me.com>
Date: Mon, 17 Jun 2019 12:20:36 -0700
Subject: [PATCH 15/15] Restore original intent of --dry-run

---
 tests/pytype_test.py | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index 655c0b530e14..c67e85941931 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -33,8 +33,6 @@ def main() -> None:
     subdir_paths = [os.path.join(typeshed_location, d) for d in TYPESHED_SUBDIRS]
     check_subdirs_discoverable(subdir_paths)
     check_python_exes_runnable(python27_exe_arg=args.python27_exe, python36_exe_arg=args.python36_exe)
-    if args.dry_run:
-        return
     files_to_test = determine_files_to_test(typeshed_location=typeshed_location, subdir_paths=subdir_paths)
     run_all_tests(
         files_to_test=files_to_test,
@@ -42,6 +40,7 @@ def main() -> None:
         python27_exe=args.python27_exe,
         python36_exe=args.python36_exe,
         print_stderr=args.print_stderr,
+        dry_run=args.dry_run,
     )
 
 
@@ -184,18 +183,28 @@ def determine_files_to_test(*, typeshed_location: str, subdir_paths: Sequence[st
 
 
 def run_all_tests(
-    *, files_to_test: Sequence[Tuple[str, int]], typeshed_location: str, python27_exe: str, python36_exe: str, print_stderr: bool
+    *,
+    files_to_test: Sequence[Tuple[str, int]],
+    typeshed_location: str,
+    python27_exe: str,
+    python36_exe: str,
+    print_stderr: bool,
+    dry_run: bool
 ) -> None:
     bad = []
     errors = 0
     total_tests = len(files_to_test)
     print("Testing files with pytype...")
     for i, (f, version) in enumerate(files_to_test):
-        stderr = run_pytype(
-            filename=f,
-            python_version="2.7" if version == 2 else "3.6",
-            python_exe=python27_exe if version == 2 else python36_exe,
-            typeshed_location=typeshed_location,
+        stderr = (
+            run_pytype(
+                filename=f,
+                python_version="2.7" if version == 2 else "3.6",
+                python_exe=python27_exe if version == 2 else python36_exe,
+                typeshed_location=typeshed_location,
+            )
+            if not dry_run
+            else None
         )
         if stderr:
             if print_stderr: