From 637bddcc4c007d6f44cf5181f3f0e76e97c1db3c Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 19 Oct 2016 16:32:48 -0400 Subject: [PATCH 1/3] Add coverage reporting, uploaded to codecov.io --- .gitignore | 4 ++++ .travis.yml | 3 +++ conftest.py | 1 + pytest.ini | 2 +- runtests.py | 2 +- setup.cfg | 6 ++++++ test-requirements.txt | 1 + 7 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 97c998f3d08e8..d17d38ac3c90b 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,7 @@ docs/build/ # Operating Systems .DS_store + +# Coverage Files +htmlcov +.coverage diff --git a/.travis.yml b/.travis.yml index 146bc7b49aa29..39c1a8f9efe6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,3 +14,6 @@ install: script: - python runtests.py + +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/conftest.py b/conftest.py index 9673db23c2fcb..b2f38246b02ef 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,4 @@ pytest_plugins = [ 'mypy.test.data', + 'pytest_cov', ] diff --git a/pytest.ini b/pytest.ini index 265a79141ab32..9413d404fb680 100644 --- a/pytest.ini +++ b/pytest.ini @@ -11,4 +11,4 @@ python_classes = python_functions = # always run in parallel (requires pytest-xdist, see test-requirements.txt) -addopts = -nauto +addopts = -nauto --cov=mypy --cov-report html --cov-report= diff --git a/runtests.py b/runtests.py index c90dbc61fdc19..2e753f2565da5 100755 --- a/runtests.py +++ b/runtests.py @@ -98,7 +98,7 @@ def add_pytest(self, name: str, pytest_args: List[str]) -> None: full_name = 'pytest %s' % name if not self.allow(full_name): return - args = [sys.executable, '-m', 'pytest'] + pytest_args + args = [sys.executable, '-m', 'pytest', '--cov-append'] + pytest_args self.waiter.add(LazySubprocess(full_name, args, env=self.env)) def add_python(self, name: str, *args: str, cwd: Optional[str] = None) -> None: diff --git a/setup.cfg b/setup.cfg index ea994efe18323..1ecd5dfad7089 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,3 +11,9 @@ exclude = mypy/codec/* # E704: multiple statements on one line (def) # E402: module level import not at top of file ignore = E251,E128,F401,W601,E701,W503,E704,E402 + +[coverage:run] +branch = true + +[coverage:report] +show_missing = true diff --git a/test-requirements.txt b/test-requirements.txt index 0defc6aea1a48..ed3b707510bf4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,3 +2,4 @@ flake8 typed-ast pytest>=2.8 pytest-xdist>=1.13 +pytest-cov>=2.4.0 From 6e0f98c9570b3ec3102acfa7709f474277d1615a Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 20 Oct 2016 16:53:09 -0400 Subject: [PATCH 2/3] Meausure coverage of myunit tests --- .gitignore | 2 +- .travis.yml | 2 ++ pytest.ini | 2 +- runtests.py | 19 +++++++++++++------ setup.cfg | 2 ++ 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index d17d38ac3c90b..731180e6e1634 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,4 @@ docs/build/ # Coverage Files htmlcov -.coverage +.coverage* diff --git a/.travis.yml b/.travis.yml index 39c1a8f9efe6c..cda6df2fee7bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,9 @@ install: - python setup.py install script: + - coverage erase - python runtests.py + - coverage combine after_success: - bash <(curl -s https://codecov.io/bash) diff --git a/pytest.ini b/pytest.ini index 9413d404fb680..cc553081ca9eb 100644 --- a/pytest.ini +++ b/pytest.ini @@ -11,4 +11,4 @@ python_classes = python_functions = # always run in parallel (requires pytest-xdist, see test-requirements.txt) -addopts = -nauto --cov=mypy --cov-report html --cov-report= +addopts = -nauto --cov=mypy --cov-report= diff --git a/runtests.py b/runtests.py index 2e753f2565da5..f12b2c8ae01cf 100755 --- a/runtests.py +++ b/runtests.py @@ -98,7 +98,7 @@ def add_pytest(self, name: str, pytest_args: List[str]) -> None: full_name = 'pytest %s' % name if not self.allow(full_name): return - args = [sys.executable, '-m', 'pytest', '--cov-append'] + pytest_args + args = [sys.executable, '-m', 'pytest'] + pytest_args self.waiter.add(LazySubprocess(full_name, args, env=self.env)) def add_python(self, name: str, *args: str, cwd: Optional[str] = None) -> None: @@ -110,12 +110,16 @@ def add_python(self, name: str, *args: str, cwd: Optional[str] = None) -> None: env = self.env self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) - def add_python_mod(self, name: str, *args: str, cwd: Optional[str] = None) -> None: + def add_python_mod(self, name: str, *args: str, cwd: Optional[str] = None, + coverage: bool = False) -> None: name = 'run %s' % name if not self.allow(name): return largs = list(args) - largs[0:0] = [sys.executable, '-m'] + if coverage: + largs[0:0] = ['coverage', 'run', '-m'] + else: + largs[0:0] = [sys.executable, '-m'] env = self.env self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) @@ -218,17 +222,20 @@ def add_myunit(driver: Driver) -> None: # This module has been converted to pytest; don't try to use myunit. pass else: - driver.add_python_mod('unit-test %s' % mod, 'mypy.myunit', '-m', mod, *driver.arglist) + driver.add_python_mod('unit-test %s' % mod, 'mypy.myunit', '-m', mod, + *driver.arglist, coverage=True) def add_pythoneval(driver: Driver) -> None: driver.add_python_mod('eval-test', 'mypy.myunit', - '-m', 'mypy.test.testpythoneval', *driver.arglist) + '-m', 'mypy.test.testpythoneval', *driver.arglist, + coverage=True) def add_cmdline(driver: Driver) -> None: driver.add_python_mod('cmdline-test', 'mypy.myunit', - '-m', 'mypy.test.testcmdline', *driver.arglist) + '-m', 'mypy.test.testcmdline', *driver.arglist, + coverage=True) def add_stubs(driver: Driver) -> None: diff --git a/setup.cfg b/setup.cfg index 1ecd5dfad7089..dfa2885242064 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,6 +14,8 @@ ignore = E251,E128,F401,W601,E701,W503,E704,E402 [coverage:run] branch = true +source = mypy +parallel = true [coverage:report] show_missing = true From a80badbc9590b301eb776bf1ab84f056a5c65f4c Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Fri, 21 Oct 2016 15:18:08 -0400 Subject: [PATCH 3/3] Make coverage measurements optional --- .travis.yml | 2 +- pytest.ini | 2 +- runtests.py | 21 +++++++++++++++------ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index cda6df2fee7bb..1e6482a0ad08b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ install: script: - coverage erase - - python runtests.py + - python runtests.py --coverage - coverage combine after_success: diff --git a/pytest.ini b/pytest.ini index cc553081ca9eb..2b142889792b8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -11,4 +11,4 @@ python_classes = python_functions = # always run in parallel (requires pytest-xdist, see test-requirements.txt) -addopts = -nauto --cov=mypy --cov-report= +addopts = -nauto --cov-append --cov-report= diff --git a/runtests.py b/runtests.py index f12b2c8ae01cf..be9f147b71424 100755 --- a/runtests.py +++ b/runtests.py @@ -40,7 +40,7 @@ class Driver: def __init__(self, whitelist: List[str], blacklist: List[str], arglist: List[str], verbosity: int, parallel_limit: int, - xfail: List[str]) -> None: + xfail: List[str], coverage: bool) -> None: self.whitelist = whitelist self.blacklist = blacklist self.arglist = arglist @@ -50,6 +50,7 @@ def __init__(self, whitelist: List[str], blacklist: List[str], self.cwd = os.getcwd() self.mypy = os.path.join(self.cwd, 'scripts', 'mypy') self.env = dict(os.environ) + self.coverage = coverage def prepend_path(self, name: str, paths: List[str]) -> None: old_val = self.env.get(name) @@ -94,11 +95,15 @@ def add_mypy_package(self, name: str, packagename: str, *flags: str) -> None: def add_mypy_string(self, name: str, *args: str, cwd: Optional[str] = None) -> None: self.add_mypy_cmd(name, ['-c'] + list(args), cwd=cwd) - def add_pytest(self, name: str, pytest_args: List[str]) -> None: + def add_pytest(self, name: str, pytest_args: List[str], coverage: bool = False) -> None: full_name = 'pytest %s' % name if not self.allow(full_name): return - args = [sys.executable, '-m', 'pytest'] + pytest_args + if coverage and self.coverage: + args = [sys.executable, '-m', 'pytest', '--cov=mypy'] + pytest_args + else: + args = [sys.executable, '-m', 'pytest'] + pytest_args + self.waiter.add(LazySubprocess(full_name, args, env=self.env)) def add_python(self, name: str, *args: str, cwd: Optional[str] = None) -> None: @@ -116,7 +121,7 @@ def add_python_mod(self, name: str, *args: str, cwd: Optional[str] = None, if not self.allow(name): return largs = list(args) - if coverage: + if coverage and self.coverage: largs[0:0] = ['coverage', 'run', '-m'] else: largs[0:0] = [sys.executable, '-m'] @@ -207,7 +212,7 @@ def add_imports(driver: Driver) -> None: def add_pytest(driver: Driver) -> None: for f in PYTEST_FILES: - driver.add_pytest(f, [f] + driver.arglist) + driver.add_pytest(f, [f] + driver.arglist, True) def add_myunit(driver: Driver) -> None: @@ -295,6 +300,7 @@ def usage(status: int) -> None: print(' -l, --list list included tasks (after filtering) and exit') print(' FILTER include tasks matching FILTER') print(' -x, --exclude FILTER exclude tasks matching FILTER') + print(' -c, --coverage calculate code coverage while running tests') print(' -- treat all remaining arguments as positional') sys.exit(status) @@ -323,6 +329,7 @@ def main() -> None: blacklist = [] # type: List[str] arglist = [] # type: List[str] list_only = False + coverage = False allow_opts = True curlist = whitelist @@ -347,6 +354,8 @@ def main() -> None: curlist = arglist elif a == '-l' or a == '--list': list_only = True + elif a == '-c' or a == '--coverage': + coverage = True elif a == '-h' or a == '--help': usage(0) else: @@ -363,7 +372,7 @@ def main() -> None: whitelist.append('') driver = Driver(whitelist=whitelist, blacklist=blacklist, arglist=arglist, - verbosity=verbosity, parallel_limit=parallel_limit, xfail=[]) + verbosity=verbosity, parallel_limit=parallel_limit, xfail=[], coverage=coverage) driver.prepend_path('PATH', [join(driver.cwd, 'scripts')]) driver.prepend_path('MYPYPATH', [driver.cwd])