diff --git a/mypy/options.py b/mypy/options.py
index 667d88065648..5ea251df2c9d 100644
--- a/mypy/options.py
+++ b/mypy/options.py
@@ -52,7 +52,7 @@ def __init__(self) -> None:
 
         # -- build options --
         self.build_type = BuildType.STANDARD
-        self.python_version = defaults.PYTHON3_VERSION
+        self.python_version = sys.version_info[:2]  # type: Tuple[int, int]
         self.platform = sys.platform
         self.custom_typing_module = None  # type: Optional[str]
         self.custom_typeshed_dir = None  # type: Optional[str]
diff --git a/mypy/test/testdiff.py b/mypy/test/testdiff.py
index b1cfc65a4a29..b13c8402befa 100644
--- a/mypy/test/testdiff.py
+++ b/mypy/test/testdiff.py
@@ -5,6 +5,7 @@
 
 from mypy import build
 from mypy.build import BuildSource
+from mypy.defaults import PYTHON3_VERSION
 from mypy.errors import CompileError
 from mypy.nodes import MypyFile
 from mypy.options import Options
@@ -53,6 +54,7 @@ def build(self, source: str) -> Tuple[List[str], Optional[Dict[str, MypyFile]]]:
         options.use_builtins_fixtures = True
         options.show_traceback = True
         options.cache_dir = os.devnull
+        options.python_version = PYTHON3_VERSION
         try:
             result = build.build(sources=[BuildSource('main', None, source)],
                                  options=options,
diff --git a/mypy/test/testmerge.py b/mypy/test/testmerge.py
index d2feece42cac..ae79e9a24747 100644
--- a/mypy/test/testmerge.py
+++ b/mypy/test/testmerge.py
@@ -6,6 +6,7 @@
 
 from mypy import build
 from mypy.build import BuildManager, BuildSource, State, Graph
+from mypy.defaults import PYTHON3_VERSION
 from mypy.errors import Errors, CompileError
 from mypy.nodes import (
     Node, MypyFile, SymbolTable, SymbolTableNode, TypeInfo, Expression, Var, TypeVarExpr,
@@ -106,6 +107,7 @@ def build(self, source: str) -> Tuple[List[str], Optional[BuildManager], Dict[st
         options.fine_grained_incremental = True
         options.use_builtins_fixtures = True
         options.show_traceback = True
+        options.python_version = PYTHON3_VERSION
         main_path = os.path.join(test_temp_dir, 'main')
         with open(main_path, 'w') as f:
             f.write(source)
diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py
index 222fa6ff32c2..bd9b1009846f 100644
--- a/mypy/test/testpythoneval.py
+++ b/mypy/test/testpythoneval.py
@@ -19,6 +19,7 @@
 import pytest  # type: ignore  # no pytest in typeshed
 from typing import Dict, List, Tuple, Optional
 
+from mypy.defaults import PYTHON3_VERSION
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal
@@ -60,6 +61,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
             return
     else:
         interpreter = python3_path
+        mypy_cmdline.append('--python-version={}'.format('.'.join(map(str, PYTHON3_VERSION))))
 
     # Write the program to a file.
     program = '_' + testcase.name + '.py'
diff --git a/mypy/test/testsemanal.py b/mypy/test/testsemanal.py
index 98f3ef64b26c..5ab772707ed6 100644
--- a/mypy/test/testsemanal.py
+++ b/mypy/test/testsemanal.py
@@ -6,6 +6,7 @@
 
 from mypy import build
 from mypy.build import BuildSource
+from mypy.defaults import PYTHON3_VERSION
 from mypy.test.helpers import (
     assert_string_arrays_equal, normalize_error_messages, testfile_pyversion,
 )
@@ -38,6 +39,7 @@ def get_semanal_options() -> Options:
     options.use_builtins_fixtures = True
     options.semantic_analysis_only = True
     options.show_traceback = True
+    options.python_version = PYTHON3_VERSION
     return options
 
 
diff --git a/mypy/waiter.py b/mypy/waiter.py
index 7e475d3e61ae..76d6fd335b0b 100644
--- a/mypy/waiter.py
+++ b/mypy/waiter.py
@@ -160,7 +160,7 @@ def load_log_file(self) -> Optional[List[Dict[str, Dict[str, Any]]]]:
                 test_log = json.load(fp)
         except FileNotFoundError:
             test_log = []
-        except json.JSONDecodeError:
+        except ValueError:
             print('corrupt test log file {}'.format(self.FULL_LOG_FILENAME), file=sys.stderr)
             test_log = []
         return test_log
diff --git a/runtests.py b/runtests.py
index c2f3361eadf5..a2a24c29a7ca 100755
--- a/runtests.py
+++ b/runtests.py
@@ -78,9 +78,10 @@ def add_mypy_cmd(self, name: str, mypy_args: List[str], cwd: Optional[str] = Non
     def add_mypy(self, name: str, *args: str, cwd: Optional[str] = None) -> None:
         self.add_mypy_cmd(name, list(args), cwd=cwd)
 
-    def add_mypy_modules(self, name: str, modules: Iterable[str],
-                         cwd: Optional[str] = None) -> None:
-        args = list(itertools.chain(*(['-m', mod] for mod in modules)))
+    def add_mypy_modules(self, name: str, modules: Iterable[str], cwd: Optional[str] = None,
+                         extra_args: Optional[List[str]] = None) -> None:
+        args = extra_args or []
+        args.extend(list(itertools.chain(*(['-m', mod] for mod in modules))))
         self.add_mypy_cmd(name, args, cwd=cwd)
 
     def add_mypy_package(self, name: str, packagename: str, *flags: str) -> None:
@@ -256,7 +257,8 @@ def add_stubs(driver: Driver) -> None:
                 module = file_to_module(f[len(stubdir) + 1:])
                 modules.add(module)
 
-    driver.add_mypy_modules('stubs', sorted(modules))
+    # these require at least 3.5 otherwise it will fail trying to import zipapp
+    driver.add_mypy_modules('stubs', sorted(modules), extra_args=['--python-version=3.5'])
 
 
 def add_stdlibsamples(driver: Driver) -> None:
@@ -276,7 +278,11 @@ def add_stdlibsamples(driver: Driver) -> None:
 
 def add_samples(driver: Driver) -> None:
     for f in find_files(os.path.join('test-data', 'samples'), suffix='.py'):
-        driver.add_mypy('file %s' % f, f)
+        if f == os.path.join('test-data', 'samples', 'crawl2.py'):
+            # This test requires 3.5 for async functions
+            driver.add_mypy_cmd('file {}'.format(f), ['--python-version=3.5', f])
+        else:
+            driver.add_mypy('file %s' % f, f)
 
 
 def usage(status: int) -> None:
diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test
index 86aae2d344f4..4becdb18e23f 100644
--- a/test-data/unit/cmdline.test
+++ b/test-data/unit/cmdline.test
@@ -581,7 +581,7 @@ m.py:6: error: Explicit "Any" is not allowed
 m.py:9: error: Explicit "Any" is not allowed
 
 [case testDisallowAnyExplicitVarDeclaration]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 
 [file mypy.ini]
 [[mypy]
@@ -601,7 +601,7 @@ m.py:3: error: Explicit "Any" is not allowed
 m.py:5: error: Explicit "Any" is not allowed
 
 [case testDisallowAnyExplicitGenericVarDeclaration]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 
 [file mypy.ini]
 [[mypy]
@@ -785,7 +785,7 @@ N = TypedDict('N', {'x': str, 'y': List})  # no error
 m.py:4: error: Explicit "Any" is not allowed
 
 [case testDisallowAnyGenericsTupleNoTypeParams]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 [file mypy.ini]
 [[mypy]
 [[mypy-m]
@@ -821,7 +821,7 @@ def g(s: List[Tuple[str, str]]) -> None: pass  # no error
 m.py:3: error: Missing type parameters for generic type
 
 [case testDisallowAnyGenericsTypeType]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 [file mypy.ini]
 [[mypy]
 [[mypy-m]
@@ -858,7 +858,7 @@ def g(l: L[str]) -> None: pass  # no error
 m.py:5: error: Missing type parameters for generic type
 
 [case testDisallowAnyGenericsGenericAlias]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 [file mypy.ini]
 [[mypy]
 [[mypy-m]
@@ -882,7 +882,7 @@ m.py:7: error: Missing type parameters for generic type
 m.py:11: error: Missing type parameters for generic type
 
 [case testDisallowAnyGenericsPlainList]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 [file mypy.ini]
 [[mypy]
 [[mypy-m]
@@ -906,7 +906,7 @@ m.py:8: error: Need type annotation for 'x'
 m.py:9: error: Missing type parameters for generic type
 
 [case testDisallowAnyGenericsCustomGenericClass]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 [file mypy.ini]
 [[mypy]
 [[mypy-m]
diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test
index 8343212fc8df..e69157d20c21 100644
--- a/test-data/unit/reports.test
+++ b/test-data/unit/reports.test
@@ -281,7 +281,7 @@ Total      0      16    100.00%
 
 
 [case testAnyExpressionsReportTypesOfAny]
-# cmd: mypy --any-exprs-report report n.py
+# cmd: mypy --python-version=3.6 --any-exprs-report report n.py
 
 [file n.py]
 from typing import Any, List