diff --git a/.gitignore b/.gitignore index 97c998f3d08e8..731180e6e1634 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,7 @@ docs/build/ # Operating Systems .DS_store + +# Coverage Files +htmlcov +.coverage* diff --git a/.travis.yml b/.travis.yml index 146bc7b49aa29..1e6482a0ad08b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,4 +13,9 @@ install: - python setup.py install script: - - python runtests.py + - coverage erase + - python runtests.py --coverage + - coverage combine + +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/conftest.py b/conftest.py index 9673db23c2fcb..b2f38246b02ef 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,4 @@ pytest_plugins = [ 'mypy.test.data', + 'pytest_cov', ] diff --git a/pytest.ini b/pytest.ini index 265a79141ab32..2b142889792b8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -11,4 +11,4 @@ python_classes = python_functions = # always run in parallel (requires pytest-xdist, see test-requirements.txt) -addopts = -nauto +addopts = -nauto --cov-append --cov-report= diff --git a/runtests.py b/runtests.py index c90dbc61fdc19..be9f147b71424 100755 --- a/runtests.py +++ b/runtests.py @@ -40,7 +40,7 @@ class Driver: def __init__(self, whitelist: List[str], blacklist: List[str], arglist: List[str], verbosity: int, parallel_limit: int, - xfail: List[str]) -> None: + xfail: List[str], coverage: bool) -> None: self.whitelist = whitelist self.blacklist = blacklist self.arglist = arglist @@ -50,6 +50,7 @@ def __init__(self, whitelist: List[str], blacklist: List[str], self.cwd = os.getcwd() self.mypy = os.path.join(self.cwd, 'scripts', 'mypy') self.env = dict(os.environ) + self.coverage = coverage def prepend_path(self, name: str, paths: List[str]) -> None: old_val = self.env.get(name) @@ -94,11 +95,15 @@ def add_mypy_package(self, name: str, packagename: str, *flags: str) -> None: def add_mypy_string(self, name: str, *args: str, cwd: Optional[str] = None) -> None: self.add_mypy_cmd(name, ['-c'] + list(args), cwd=cwd) - def add_pytest(self, name: str, pytest_args: List[str]) -> None: + def add_pytest(self, name: str, pytest_args: List[str], coverage: bool = False) -> None: full_name = 'pytest %s' % name if not self.allow(full_name): return - args = [sys.executable, '-m', 'pytest'] + pytest_args + if coverage and self.coverage: + args = [sys.executable, '-m', 'pytest', '--cov=mypy'] + pytest_args + else: + args = [sys.executable, '-m', 'pytest'] + pytest_args + self.waiter.add(LazySubprocess(full_name, args, env=self.env)) def add_python(self, name: str, *args: str, cwd: Optional[str] = None) -> None: @@ -110,12 +115,16 @@ def add_python(self, name: str, *args: str, cwd: Optional[str] = None) -> None: env = self.env self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) - def add_python_mod(self, name: str, *args: str, cwd: Optional[str] = None) -> None: + def add_python_mod(self, name: str, *args: str, cwd: Optional[str] = None, + coverage: bool = False) -> None: name = 'run %s' % name if not self.allow(name): return largs = list(args) - largs[0:0] = [sys.executable, '-m'] + if coverage and self.coverage: + largs[0:0] = ['coverage', 'run', '-m'] + else: + largs[0:0] = [sys.executable, '-m'] env = self.env self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) @@ -203,7 +212,7 @@ def add_imports(driver: Driver) -> None: def add_pytest(driver: Driver) -> None: for f in PYTEST_FILES: - driver.add_pytest(f, [f] + driver.arglist) + driver.add_pytest(f, [f] + driver.arglist, True) def add_myunit(driver: Driver) -> None: @@ -218,17 +227,20 @@ def add_myunit(driver: Driver) -> None: # This module has been converted to pytest; don't try to use myunit. pass else: - driver.add_python_mod('unit-test %s' % mod, 'mypy.myunit', '-m', mod, *driver.arglist) + driver.add_python_mod('unit-test %s' % mod, 'mypy.myunit', '-m', mod, + *driver.arglist, coverage=True) def add_pythoneval(driver: Driver) -> None: driver.add_python_mod('eval-test', 'mypy.myunit', - '-m', 'mypy.test.testpythoneval', *driver.arglist) + '-m', 'mypy.test.testpythoneval', *driver.arglist, + coverage=True) def add_cmdline(driver: Driver) -> None: driver.add_python_mod('cmdline-test', 'mypy.myunit', - '-m', 'mypy.test.testcmdline', *driver.arglist) + '-m', 'mypy.test.testcmdline', *driver.arglist, + coverage=True) def add_stubs(driver: Driver) -> None: @@ -288,6 +300,7 @@ def usage(status: int) -> None: print(' -l, --list list included tasks (after filtering) and exit') print(' FILTER include tasks matching FILTER') print(' -x, --exclude FILTER exclude tasks matching FILTER') + print(' -c, --coverage calculate code coverage while running tests') print(' -- treat all remaining arguments as positional') sys.exit(status) @@ -316,6 +329,7 @@ def main() -> None: blacklist = [] # type: List[str] arglist = [] # type: List[str] list_only = False + coverage = False allow_opts = True curlist = whitelist @@ -340,6 +354,8 @@ def main() -> None: curlist = arglist elif a == '-l' or a == '--list': list_only = True + elif a == '-c' or a == '--coverage': + coverage = True elif a == '-h' or a == '--help': usage(0) else: @@ -356,7 +372,7 @@ def main() -> None: whitelist.append('') driver = Driver(whitelist=whitelist, blacklist=blacklist, arglist=arglist, - verbosity=verbosity, parallel_limit=parallel_limit, xfail=[]) + verbosity=verbosity, parallel_limit=parallel_limit, xfail=[], coverage=coverage) driver.prepend_path('PATH', [join(driver.cwd, 'scripts')]) driver.prepend_path('MYPYPATH', [driver.cwd]) diff --git a/setup.cfg b/setup.cfg index ea994efe18323..dfa2885242064 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,3 +11,11 @@ exclude = mypy/codec/* # E704: multiple statements on one line (def) # E402: module level import not at top of file ignore = E251,E128,F401,W601,E701,W503,E704,E402 + +[coverage:run] +branch = true +source = mypy +parallel = true + +[coverage:report] +show_missing = true diff --git a/test-requirements.txt b/test-requirements.txt index 0defc6aea1a48..ed3b707510bf4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,3 +2,4 @@ flake8 typed-ast pytest>=2.8 pytest-xdist>=1.13 +pytest-cov>=2.4.0