From fda8ee66fecc807d60988691ded6ae4514af341b Mon Sep 17 00:00:00 2001 From: Drew Hubl Date: Fri, 25 May 2018 12:28:27 -0600 Subject: [PATCH 1/5] Rewrite coverage handling in the root Makefile --- .coveragerc | 2 +- Makefile | 173 ++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 148 insertions(+), 27 deletions(-) diff --git a/.coveragerc b/.coveragerc index 49bf8fe043..b212365101 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,4 +2,4 @@ [run] include = st2* -omit = *tests/* +disable_warnings = include-ignored,module-not-imported diff --git a/Makefile b/Makefile index cf130e9042..b80b0cb889 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,14 @@ ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) SHELL := /bin/bash TOX_DIR := .tox -VIRTUALENV_DIR ?= virtualenv +OS := $(shell uname) +# We separate the OSX X and Linux virtualenvs so we can run in a Docker +# container (st2devbox) while doing things on our host Mac machine +ifeq ($(OS),Darwin) + VIRTUALENV_DIR ?= virtualenv-osx +else + VIRTUALENV_DIR ?= virtualenv +endif PYTHON_VERSION = python2.7 BINARIES := bin @@ -10,8 +17,9 @@ BINARIES := bin COMPONENTS := $(wildcard st2*) COMPONENTS_RUNNERS := $(wildcard contrib/runners/*) -COMPONENTS_WITH_RUNNERS := $(wildcard st2*) -COMPONENTS_WITH_RUNNERS += $(wildcard contrib/runners/*) +COMPONENTS_WITH_RUNNERS := $(wildcard st2*) $(COMPONENTS_RUNNERS) + +COMPONENTS_TEST_DIRS := $(wildcard st2*/tests) $(wildcard contrib/runners/*/tests) # Components that implement a component-controlled test-runner. These components provide an # in-component Makefile. (Temporary fix until I can generalize the pecan unittest setup. -mar) @@ -19,12 +27,17 @@ COMPONENTS_WITH_RUNNERS += $(wildcard contrib/runners/*) COMPONENT_SPECIFIC_TESTS := st2tests st2client.egg-info # nasty hack to get a space into a variable +colon := : +comma := , +dot := . +slash := / space_char := space_char += -comma := , COMPONENT_PYTHONPATH = $(subst $(space_char),:,$(realpath $(COMPONENTS_WITH_RUNNERS))) COMPONENTS_TEST := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS_WITH_RUNNERS)),$(component)) -COMPONENTS_TEST_COMMA := $(subst $(space_char),$(comma),$(COMPONENTS_TEST)) +COMPONENTS_TEST_COMMA := $(subst $(slash),$(dot),$(subst $(space_char),$(comma),$(COMPONENTS_TEST))) +COMPONENTS_TEST_MODULES := $(subst $(slash),$(dot),$(COMPONENTS_TEST_DIRS)) +COMPONENTS_TEST_MODULES_COMMA := $(subst $(space_char),$(comma),$(COMPONENTS_TEST_MODULES)) PYTHON_TARGET := 2.7 @@ -33,6 +46,11 @@ PIP_OPTIONS := $(ST2_PIP_OPTIONS) NOSE_OPTS := --rednose --immediate --with-parallel NOSE_TIME := $(NOSE_TIME) +NOSE_COVERAGE_FLAGS := --with-coverage --cover-branches --cover-erase --cover-tests +NOSE_COVERAGE_PACKAGES := --cover-package=$(COMPONENTS_TEST_COMMA),$(COMPONENTS_TEST_MODULES_COMMA) +# --coverage-inclusive + +COVERAGE_FLAGS := --branch --concurrency=eventlet --source=$(COMPONENTS_TEST_COMMA),$(COMPONENTS_TEST_MODULES_COMMA) ifdef NOSE_TIME NOSE_OPTS := --rednose --immediate --with-parallel --with-timer @@ -49,11 +67,23 @@ all: requirements configgen check tests .PHONY: play play: @echo COMPONENTS=$(COMPONENTS) + @echo @echo COMPONENTS_WITH_RUNNERS=$(COMPONENTS_WITH_RUNNERS) + @echo @echo COMPONENTS_TEST=$(COMPONENTS_TEST) + @echo @echo COMPONENTS_TEST_COMMA=$(COMPONENTS_TEST_COMMA) + @echo + @echo COMPONENTS_TEST_DIRS=$(COMPONENTS_TEST_DIRS) + @echo + @echo COMPONENTS_TEST_MODULES=$(COMPONENTS_TEST_MODULES) + @echo + @echo COMPONENTS_TEST_MODULES_COMMA=$(COMPONENTS_TEST_MODULES_COMMA) + @echo @echo COMPONENT_PYTHONPATH=$(COMPONENT_PYTHONPATH) - + @echo + @echo NOSE_COVERAGE_FLAGS=$(NOSE_COVERAGE_FLAGS) + @echo .PHONY: check check: requirements flake8 checklogs @@ -241,6 +271,16 @@ compilepy3: @sudo rabbitmqctl reset @sudo rabbitmqctl start_app +.PHONY: .cleancoverage +.cleancoverage: + @echo "==================== cleancoverage ====================" + @echo "Removing all coverage results directories" + @echo + rm -rf .coverage \ + .coverage-unit .coverage-unit-* \ + .coverage-integation .coverage-integration-* \ + .coverage-integration-mistal + .PHONY: distclean distclean: clean @echo @@ -254,6 +294,7 @@ requirements: virtualenv .sdist-requirements @echo "==================== requirements ====================" @echo # Make sure we use latest version of pip which is < 10.0.0 + $(VIRTUALENV_DIR)/bin/pip --version $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip>=9.0,<9.1" $(VIRTUALENV_DIR)/bin/pip install --upgrade "virtualenv==15.1.0" # Required for packs.install in dev envs. @@ -356,26 +397,53 @@ unit-tests: requirements .unit-tests @for component in $(COMPONENTS_TEST); do\ echo "==========================================================="; \ echo "Running tests in" $$component; \ + echo "-----------------------------------------------------------"; \ + . $(VIRTUALENV_DIR)/bin/activate; \ + nosetests $(NOSE_OPTS) -s -v \ + $$component/tests/unit || exit 1; \ + echo "-----------------------------------------------------------"; \ + echo "Done running tests in" $$component; \ echo "==========================================================="; \ - . $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v $$component/tests/unit || exit 1; \ done -.PHONY: .unit-tests-coverage-html -.unit-tests-coverage-html: +.PHONY: .run-unit-tests-coverage +.run-unit-tests-coverage: @echo - @echo "==================== unit tests with coverage (HTML reports) ====================" + @echo "==================== unit tests with coverage ====================" @echo @echo "----- Dropping st2-test db -----" @mongo st2-test --eval "db.dropDatabase();" - @for component in $(COMPONENTS_TEST); do\ + for component in $(COMPONENTS_TEST); do\ echo "==========================================================="; \ echo "Running tests in" $$component; \ + echo "-----------------------------------------------------------"; \ + . $(VIRTUALENV_DIR)/bin/activate; \ + COVERAGE_FILE=.coverage-unit-$$(echo $$component | tr '/' '.') \ + nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \ + $(NOSE_COVERAGE_PACKAGES),tests.unit \ + $$component/tests/unit || exit 1; \ + echo "-----------------------------------------------------------"; \ + echo "Done running tests in" $$component; \ echo "==========================================================="; \ - . $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v --with-coverage \ - --cover-inclusive --cover-html \ - --cover-package=$(COMPONENTS_TEST_COMMA) $$component/tests/unit || exit 1; \ done +.PHONY: .combine-unit-tests-coverage +.combine-unit-tests-coverage: .run-unit-tests-coverage + . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-unit \ + coverage combine .coverage-unit-* + +.coverage-unit: .combine-unit-tests-coverage + +.PHONY: .report-unit-tests-coverage +.report-unit-tests-coverage: .coverage-unit + . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-unit \ + coverage report + +.PHONY: .unit-tests-coverage-html +.unit-tests-coverage-html: .coverage-unit + . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-unit \ + coverage html + .PHONY: itests itests: requirements .itests @@ -389,26 +457,56 @@ itests: requirements .itests @for component in $(COMPONENTS_TEST); do\ echo "==========================================================="; \ echo "Running tests in" $$component; \ + echo "-----------------------------------------------------------"; \ + . $(VIRTUALENV_DIR)/bin/activate; \ + nosetests $(NOSE_OPTS) -s -v \ + $$component/tests/integration || exit 1; \ + echo "-----------------------------------------------------------"; \ + echo "Done running tests in" $$component; \ echo "==========================================================="; \ - . $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v $$component/tests/integration || exit 1; \ done -.PHONY: .itests-coverage-html -.itests-coverage-html: +.PHONY: .run-integration-tests-coverage +.run-integration-tests-coverage: @echo - @echo "================ integration tests with coverage (HTML reports) ================" + @echo "================ integration tests with coverage ================" @echo @echo "----- Dropping st2-test db -----" @mongo st2-test --eval "db.dropDatabase();" @for component in $(COMPONENTS_TEST); do\ echo "==========================================================="; \ echo "Running tests in" $$component; \ + echo "-----------------------------------------------------------"; \ + . $(VIRTUALENV_DIR)/bin/activate; \ + COVERAGE_FILE=.coverage-integration-$$(echo $$component | tr '/' '.') \ + nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \ + $(NOSE_COVERAGE_PACKAGES),tests.integration \ + $$component/tests/integration || exit 1; \ + echo "-----------------------------------------------------------"; \ + echo "Done running tests in" $$component; \ echo "==========================================================="; \ - . $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v --with-coverage \ - --cover-inclusive --cover-html \ - --cover-package=$(COMPONENTS_TEST_COMMA) $$component/tests/integration || exit 1; \ done +.PHONY: .combine-integration-tests-coverage +.combine-integration-tests-coverage: .run-integration-tests-coverage + . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-integration \ + coverage combine .coverage-integration-* + +.coverage-integration: .combine-integration-tests-coverage + +.PHONY: .report-integration-tests-coverage +.report-integration-tests-coverage: .coverage-integration + @. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-integration \ + coverage report + +.PHONY: .integration-tests-coverage-html +.integration-tests-coverage-html: .coverage-integration + @. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-integration \ + coverage html + +.PHONY: .itests-coverage-html +.itests-coverage-html: .integration-tests-coverage-html + .PHONY: mistral-itests mistral-itests: requirements .mistral-itests @@ -420,14 +518,37 @@ mistral-itests: requirements .mistral-itests @echo . $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v st2tests/integration/mistral || exit 1; -.PHONY: .mistral-itests-coverage-html -.mistral-itests-coverage-html: +.PHONY: .run-mistral-itests-coverage +.run-mistral-itests-coverage: @echo - @echo "==================== MISTRAL integration tests with coverage (HTML reports) ====================" + @echo "==================== MISTRAL integration tests with coverage ====================" @echo "The tests assume both st2 and mistral are running on 127.0.0.1." @echo - . $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v --with-coverage \ - --cover-inclusive --cover-html st2tests/integration/mistral || exit 1; + . $(VIRTUALENV_DIR)/bin/activate; \ + COVERAGE_FILE=.coverage-integration-mistal \ + nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \ + $(NOSE_COVERAGE_PACKAGES),st2tests.integration.mistal \ + st2tests/integration/mistral || exit 1; + +.coverage-integration-mistal: .run-mistral-itests-coverage + +.PHONY: .mistral-itests-coverage-html +.mistral-itests-coverage-html: .run-mistral-itests-coverage + . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-integration-mistal coverage html + +.PHONY: .coverage-combine +.coverage-combine: .run-unit-tests-coverage .run-integration-tests-coverage .run-mistral-itests-coverage + . $(VIRTUALENV_DIR)/bin/activate; coverage combine .coverage-unit-* .coverage-integration-* .coverage-integration-mistal + +.coverage: .coverage-combine + +.PHONY: .coverage-report +.coverage-report: .coverage + . $(VIRTUALENV_DIR)/bin/activate; coverage report + +.PHONY: .coverage-html +.coverage-html: .coverage + . $(VIRTUALENV_DIR)/bin/activate; coverage html .PHONY: packs-tests packs-tests: requirements .packs-tests From 60ba76e9e86e372724b6757ae4bc1f8166066ca7 Mon Sep 17 00:00:00 2001 From: Drew Hubl Date: Tue, 29 May 2018 10:43:04 -0600 Subject: [PATCH 2/5] Quote arguments to rm --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b80b0cb889..81893e0e41 100644 --- a/Makefile +++ b/Makefile @@ -277,8 +277,8 @@ compilepy3: @echo "Removing all coverage results directories" @echo rm -rf .coverage \ - .coverage-unit .coverage-unit-* \ - .coverage-integation .coverage-integration-* \ + .coverage-unit ".coverage-unit-*" \ + .coverage-integation ".coverage-integration-*" \ .coverage-integration-mistal .PHONY: distclean From f8db82f75579c01e1cdd6532d4da66b639dafbb3 Mon Sep 17 00:00:00 2001 From: Drew Hubl Date: Fri, 1 Jun 2018 00:12:19 -0600 Subject: [PATCH 3/5] Use .coveragerc to control covered files, and omit certain files from coverage --- .coveragerc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index b212365101..eb009001ce 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,13 @@ ; http://nedbatchelder.com/code/coverage/config.html [run] -include = st2* +branch = True +concurrency = eventlet +include = st2*,contrib/runners/* disable_warnings = include-ignored,module-not-imported +omit = */*/wsgi.py + */*/util/wsgi.py ; st2common/st2common/util/wsgi.py + */setup.py + contrib/runners/*/setup.py + */dist_utils.py + contrib/runners/*/dist_utils.py From 576ae76fcd8cd7c91569ddb824f92f8397bebb9b Mon Sep 17 00:00:00 2001 From: Drew Hubl Date: Wed, 6 Jun 2018 00:45:41 -0600 Subject: [PATCH 4/5] Control whether tests are included in coverage results with the INCLUDE_TESTS_IN_COVERAGE environment variable --- Makefile | 143 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 106 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index 81893e0e41..0ef1df486d 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,9 @@ COMPONENTS_TEST_COMMA := $(subst $(slash),$(dot),$(subst $(space_char),$(comma), COMPONENTS_TEST_MODULES := $(subst $(slash),$(dot),$(COMPONENTS_TEST_DIRS)) COMPONENTS_TEST_MODULES_COMMA := $(subst $(space_char),$(comma),$(COMPONENTS_TEST_MODULES)) +COVERAGE_GLOBS := .coverage.unit.* .coverage.integration.* .coverage.mistral.* +COVERAGE_GLOBS_QUOTED := $(foreach glob,$(COVERAGE_GLOBS),'$(glob)') + PYTHON_TARGET := 2.7 REQUIREMENTS := test-requirements.txt requirements.txt @@ -46,11 +49,8 @@ PIP_OPTIONS := $(ST2_PIP_OPTIONS) NOSE_OPTS := --rednose --immediate --with-parallel NOSE_TIME := $(NOSE_TIME) -NOSE_COVERAGE_FLAGS := --with-coverage --cover-branches --cover-erase --cover-tests -NOSE_COVERAGE_PACKAGES := --cover-package=$(COMPONENTS_TEST_COMMA),$(COMPONENTS_TEST_MODULES_COMMA) -# --coverage-inclusive - -COVERAGE_FLAGS := --branch --concurrency=eventlet --source=$(COMPONENTS_TEST_COMMA),$(COMPONENTS_TEST_MODULES_COMMA) +NOSE_COVERAGE_FLAGS := --with-coverage --cover-branches --cover-erase +NOSE_COVERAGE_PACKAGES := --cover-package=$(COMPONENTS_TEST_COMMA) ifdef NOSE_TIME NOSE_OPTS := --rednose --immediate --with-parallel --with-timer @@ -60,12 +60,29 @@ ifndef PIP_OPTIONS PIP_OPTIONS := endif +ifneq ($(INCLUDE_TESTS_IN_COVERAGE),) + NOSE_COVERAGE_FLAGS += --cover-tests + NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),$(COMPONENTS_TEST_MODULES_COMMA) +endif + .PHONY: all all: requirements configgen check tests +.PHONY: .coverage_globs +.coverage_globs: + @for coverage_result in $$( \ + for coverage_glob in $(COVERAGE_GLOBS_QUOTED); do \ + compgen -G $${coverage_glob}; \ + done; \ + ); do \ + echo $${coverage_result}; \ + done + # Target for debugging Makefile variable assembly .PHONY: play play: + @echo COVERAGE_GLOBS=$(COVERAGE_GLOBS_QUOTED) + @echo @echo COMPONENTS=$(COMPONENTS) @echo @echo COMPONENTS_WITH_RUNNERS=$(COMPONENTS_WITH_RUNNERS) @@ -84,6 +101,8 @@ play: @echo @echo NOSE_COVERAGE_FLAGS=$(NOSE_COVERAGE_FLAGS) @echo + @echo NOSE_COVERAGE_PACKAGES=$(NOSE_COVERAGE_PACKAGES) + @echo .PHONY: check check: requirements flake8 checklogs @@ -276,10 +295,8 @@ compilepy3: @echo "==================== cleancoverage ====================" @echo "Removing all coverage results directories" @echo - rm -rf .coverage \ - .coverage-unit ".coverage-unit-*" \ - .coverage-integation ".coverage-integration-*" \ - .coverage-integration-mistal + rm -rf .coverage $(COVERAGE_GLOBS) \ + .coverage.unit .coverage.integration .coverage.mistral .PHONY: distclean distclean: clean @@ -407,6 +424,9 @@ unit-tests: requirements .unit-tests done .PHONY: .run-unit-tests-coverage +ifneq ($(INCLUDE_TESTS_IN_COVERAGE),) +.run-unit-tests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),tests.unit +endif .run-unit-tests-coverage: @echo @echo "==================== unit tests with coverage ====================" @@ -418,9 +438,9 @@ unit-tests: requirements .unit-tests echo "Running tests in" $$component; \ echo "-----------------------------------------------------------"; \ . $(VIRTUALENV_DIR)/bin/activate; \ - COVERAGE_FILE=.coverage-unit-$$(echo $$component | tr '/' '.') \ + COVERAGE_FILE=.coverage.unit.$$(echo $$component | tr '/' '.') \ nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \ - $(NOSE_COVERAGE_PACKAGES),tests.unit \ + $(NOSE_COVERAGE_PACKAGES) \ $$component/tests/unit || exit 1; \ echo "-----------------------------------------------------------"; \ echo "Done running tests in" $$component; \ @@ -429,19 +449,28 @@ unit-tests: requirements .unit-tests .PHONY: .combine-unit-tests-coverage .combine-unit-tests-coverage: .run-unit-tests-coverage - . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-unit \ - coverage combine .coverage-unit-* - -.coverage-unit: .combine-unit-tests-coverage + . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.unit \ + coverage combine .coverage.unit.* + +.coverage.unit: + @compgen -G '.coverage.unit.*' && \ + for coverage_result in $$(compgen -G '.coverage.unit.*'); do \ + echo "Combining data from $${coverage_result}"; \ + . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.unit \ + coverage combine $${coverage_result}; \ + done \ + || \ + echo "Running unit tests"; \ + make .combine-unit-tests-coverage .PHONY: .report-unit-tests-coverage -.report-unit-tests-coverage: .coverage-unit - . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-unit \ +.report-unit-tests-coverage: .coverage.unit + . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.unit \ coverage report .PHONY: .unit-tests-coverage-html -.unit-tests-coverage-html: .coverage-unit - . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-unit \ +.unit-tests-coverage-html: .coverage.unit + . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.unit \ coverage html .PHONY: itests @@ -467,6 +496,9 @@ itests: requirements .itests done .PHONY: .run-integration-tests-coverage +ifneq ($(INCLUDE_TESTS_IN_COVERAGE),) +.run-integration-tests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),tests.integration +endif .run-integration-tests-coverage: @echo @echo "================ integration tests with coverage ================" @@ -478,9 +510,9 @@ itests: requirements .itests echo "Running tests in" $$component; \ echo "-----------------------------------------------------------"; \ . $(VIRTUALENV_DIR)/bin/activate; \ - COVERAGE_FILE=.coverage-integration-$$(echo $$component | tr '/' '.') \ + COVERAGE_FILE=.coverage.integration.$$(echo $$component | tr '/' '.') \ nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \ - $(NOSE_COVERAGE_PACKAGES),tests.integration \ + $(NOSE_COVERAGE_PACKAGES) \ $$component/tests/integration || exit 1; \ echo "-----------------------------------------------------------"; \ echo "Done running tests in" $$component; \ @@ -489,19 +521,28 @@ itests: requirements .itests .PHONY: .combine-integration-tests-coverage .combine-integration-tests-coverage: .run-integration-tests-coverage - . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-integration \ - coverage combine .coverage-integration-* - -.coverage-integration: .combine-integration-tests-coverage + . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.integration \ + coverage combine .coverage.integration.* + +.coverage.integration: + @compgen -G '.coverage.integration.*' && \ + for coverage_result in $$(compgen -G '.coverage.integration.*'); do \ + echo "Combining data from $${coverage_result}"; \ + . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.integration \ + coverage combine $${coverage_result}; \ + done \ + || \ + echo "Running integration tests"; \ + make .combine-integration-tests-coverage .PHONY: .report-integration-tests-coverage -.report-integration-tests-coverage: .coverage-integration - @. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-integration \ +.report-integration-tests-coverage: .coverage.integration + @. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.integration \ coverage report .PHONY: .integration-tests-coverage-html -.integration-tests-coverage-html: .coverage-integration - @. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-integration \ +.integration-tests-coverage-html: .coverage.integration + @. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.integration \ coverage html .PHONY: .itests-coverage-html @@ -519,28 +560,56 @@ mistral-itests: requirements .mistral-itests . $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v st2tests/integration/mistral || exit 1; .PHONY: .run-mistral-itests-coverage +ifneq ($(INCLUDE_TESTS_IN_COVERAGE),) +.run-mistral-itests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),st2tests.mistral.integration +endif .run-mistral-itests-coverage: @echo @echo "==================== MISTRAL integration tests with coverage ====================" @echo "The tests assume both st2 and mistral are running on 127.0.0.1." @echo . $(VIRTUALENV_DIR)/bin/activate; \ - COVERAGE_FILE=.coverage-integration-mistal \ + COVERAGE_FILE=.coverage.mistral.integration \ nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \ - $(NOSE_COVERAGE_PACKAGES),st2tests.integration.mistal \ + $(NOSE_COVERAGE_PACKAGES) \ st2tests/integration/mistral || exit 1; -.coverage-integration-mistal: .run-mistral-itests-coverage +.coverage.mistral.integration: + if [ ! -e .coverage.mistral.integration ]; then \ + make .run-mistral-itests-coverage; \ + fi .PHONY: .mistral-itests-coverage-html -.mistral-itests-coverage-html: .run-mistral-itests-coverage - . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage-integration-mistal coverage html +.mistral-itests-coverage-html: .coverage.mistral.integration + . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.mistral.integration coverage html .PHONY: .coverage-combine .coverage-combine: .run-unit-tests-coverage .run-integration-tests-coverage .run-mistral-itests-coverage - . $(VIRTUALENV_DIR)/bin/activate; coverage combine .coverage-unit-* .coverage-integration-* .coverage-integration-mistal - -.coverage: .coverage-combine + . $(VIRTUALENV_DIR)/bin/activate; coverage combine $(COVERAGE_GLOBS) + +# This is a real target, but we need to do our own make trickery in case some +# but not all of the prerequisites are available +.coverage: + @NUM_COVERAGE_RESULTS=0; \ + for coverage_result in $$( \ + for coverage_glob in $(COVERAGE_GLOBS_QUOTED); do \ + compgen -G $${coverage_glob}; \ + done; \ + ); do \ + NUM_COVERAGE_RESULTS=$$(( NUM_COVERAGE_RESULTS+1 )); \ + echo "Combining $${coverage_result}: $$NUM_COVERAGE_RESULTS"; \ + . $(VIRTUALENV_DIR)/bin/activate; coverage combine $${coverage_result}; \ + done; \ + if [ $${NUM_COVERAGE_RESULTS} -eq 0 ]; then \ + make .coverage-combine; \ + fi + +# @for coverage_result in $(COVERAGE_GLOBS); do \ +# [ -e $${coverage_result} ] || echo "$${coverage_result} does not exist." && continue; \ +# echo "Combining data from $${coverage_result}"; \ +# . $(VIRTUALENV_DIR)/bin/activate; coverage combine $${coverage_result}; \ +# done || \ +# (echo "Running .coverage-combine"; make .coverage-combine) .PHONY: .coverage-report .coverage-report: .coverage From bb65762fded45e71e8cfd74997e174e7edf9a076 Mon Sep 17 00:00:00 2001 From: Drew Hubl Date: Thu, 14 Jun 2018 18:04:05 -0700 Subject: [PATCH 5/5] Fix pause and resume ActionChain tests --- .../tests/unit/test_actionchain_pause_resume.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py index 5183967235..be40a8c09e 100644 --- a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py +++ b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py @@ -125,6 +125,8 @@ def _wait_for_status(self, liveaction, status, interval=0.1, retries=100): if liveaction.status != status: eventlet.sleep(interval) continue + else: + break return liveaction