From 2dd5b786ef4101304a3121cbd0959b3eb022b165 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Tue, 24 Jul 2018 16:59:05 +0200
Subject: [PATCH 01/15] Pull in test performance improvemetns from
 https://github.com/StackStorm/st2/pull/4263.

Changes include:

1. No need to drop collections anymore before dropping the MongoDB database
   (only needed with older versions)
2. No need to synchronously ensure indexes for tests. Async index ensure
   is fine for tests and offers significat test speeds ups.
---
 st2tests/st2tests/base.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/st2tests/st2tests/base.py b/st2tests/st2tests/base.py
index b65d769136..451f5295f5 100644
--- a/st2tests/st2tests/base.py
+++ b/st2tests/st2tests/base.py
@@ -176,6 +176,10 @@ def tearDownClass(cls):
 
 
 class BaseDbTestCase(BaseTestCase):
+    # True to synchronously ensure indexes after db_setup is called - NOTE: This is only needed
+    # with older MongoDB versions. With recent versions this is not needed for the tests anymore
+    # and offers significant test speeds ups.
+    ensure_indexes = False
 
     # Set to True to enable printing of all the log messages to the console
     DISPLAY_LOG_MESSAGES = False
@@ -196,18 +200,27 @@ def _establish_connection_and_re_create_db(cls):
         cls.db_connection = db_setup(
             cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port,
             username=username, password=password, ensure_indexes=False)
-        cls._drop_collections()
+
+        # NOTE: In older MongoDB versions you needed to drop all the collections prior to dropping
+        # the database - that's not needed anymore with the wiredtiger engine
+        # cls._drop_collections()
+
         cls.db_connection.drop_database(cfg.CONF.database.db_name)
 
         # Explicity ensure indexes after we re-create the DB otherwise ensure_indexes could failure
         # inside db_setup if test inserted invalid data
-        db_ensure_indexes()
+        if cls.ensure_indexes:
+            db_ensure_indexes()
 
     @classmethod
     def _drop_db(cls):
-        cls._drop_collections()
+        # NOTE: In older MongoDB versions you needed to drop all the collections prior to dropping
+        # the database - that's not needed anymore with the wiredtiger engine
+        # cls._drop_collections()
+
         if cls.db_connection is not None:
             cls.db_connection.drop_database(cfg.CONF.database.db_name)
+
         db_teardown()
         cls.db_connection = None
 
@@ -217,7 +230,6 @@ def _drop_collections(cls):
         # subsequent tests.
         # See: https://github.com/MongoEngine/mongoengine/issues/566
         # See: https://github.com/MongoEngine/mongoengine/issues/565
-        global ALL_MODELS
         for model in ALL_MODELS:
             model.drop_collection()
 

From f5cf5341a872496f6f42efefe87ab3f586d0898f Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Tue, 24 Jul 2018 17:12:17 +0200
Subject: [PATCH 02/15] Update affected tests.

---
 st2common/tests/unit/test_db.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/st2common/tests/unit/test_db.py b/st2common/tests/unit/test_db.py
index 4cf1e372b1..90bf4ab409 100644
--- a/st2common/tests/unit/test_db.py
+++ b/st2common/tests/unit/test_db.py
@@ -170,6 +170,7 @@ def test_db_setup_connecting_info_logging(self, mock_log, mock_mongoengine):
 
 
 class DbCleanupTest(DbTestCase):
+    ensure_indexes = True
 
     def test_cleanup(self):
         """

From b1b6b64cbc1fb46ff73e2b64d7830f9f36fe34fc Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Tue, 24 Jul 2018 17:15:49 +0200
Subject: [PATCH 03/15] Simplify the code.

---
 st2tests/st2tests/base.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/st2tests/st2tests/base.py b/st2tests/st2tests/base.py
index 451f5295f5..b3dc7fdc23 100644
--- a/st2tests/st2tests/base.py
+++ b/st2tests/st2tests/base.py
@@ -201,10 +201,7 @@ def _establish_connection_and_re_create_db(cls):
             cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port,
             username=username, password=password, ensure_indexes=False)
 
-        # NOTE: In older MongoDB versions you needed to drop all the collections prior to dropping
-        # the database - that's not needed anymore with the wiredtiger engine
-        # cls._drop_collections()
-
+        cls._drop_collections()
         cls.db_connection.drop_database(cfg.CONF.database.db_name)
 
         # Explicity ensure indexes after we re-create the DB otherwise ensure_indexes could failure
@@ -214,9 +211,7 @@ def _establish_connection_and_re_create_db(cls):
 
     @classmethod
     def _drop_db(cls):
-        # NOTE: In older MongoDB versions you needed to drop all the collections prior to dropping
-        # the database - that's not needed anymore with the wiredtiger engine
-        # cls._drop_collections()
+        cls._drop_collections()
 
         if cls.db_connection is not None:
             cls.db_connection.drop_database(cfg.CONF.database.db_name)
@@ -226,12 +221,17 @@ def _drop_db(cls):
 
     @classmethod
     def _drop_collections(cls):
-        # XXX: Explicitly drop all the collection. Otherwise, artifacts are left over in
+        # XXX: Explicitly drop all the collections. Otherwise, artifacts are left over in
         # subsequent tests.
         # See: https://github.com/MongoEngine/mongoengine/issues/566
         # See: https://github.com/MongoEngine/mongoengine/issues/565
-        for model in ALL_MODELS:
-            model.drop_collection()
+
+        # NOTE: In older MongoDB versions you needed to drop all the collections prior to dropping
+        # the database - that's not needed anymore with the WiredTiger engine
+
+        # for model in ALL_MODELS:
+        #     model.drop_collection()
+        return
 
 
 class DbTestCase(BaseDbTestCase):

From 5953071ea48d42b188215503d21c35dcd799de08 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Tue, 24 Jul 2018 17:18:23 +0200
Subject: [PATCH 04/15] Update the comment, log the message.

---
 st2tests/st2tests/base.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/st2tests/st2tests/base.py b/st2tests/st2tests/base.py
index b3dc7fdc23..2aa0c6ccca 100644
--- a/st2tests/st2tests/base.py
+++ b/st2tests/st2tests/base.py
@@ -14,6 +14,8 @@
 # limitations under the License.
 
 from __future__ import absolute_import
+from __future__ import print_function
+
 try:
     import simplejson as json
 except ImportError:
@@ -204,9 +206,16 @@ def _establish_connection_and_re_create_db(cls):
         cls._drop_collections()
         cls.db_connection.drop_database(cfg.CONF.database.db_name)
 
-        # Explicity ensure indexes after we re-create the DB otherwise ensure_indexes could failure
-        # inside db_setup if test inserted invalid data
+        # Explicitly ensure indexes after we re-create the DB otherwise ensure_indexes could failure
+        # inside db_setup if test inserted invalid data.
+        # NOTE: This is only needed in distributed scenarios (production deployments) where
+        # multiple services can start up at the same time and race conditions are possible.
         if cls.ensure_indexes:
+            msg = ('Ensuring indexes for all the models, this could significantly slow down the '
+                  'tests')
+            print('#' * len(msg), file=sys.stderr)
+            print(msg, file=sys.stderr)
+            print('#' * len(msg), file=sys.stderr)
             db_ensure_indexes()
 
     @classmethod

From 4017370c537abb33af5990256427d8a901b9b7da Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Tue, 24 Jul 2018 17:43:45 +0200
Subject: [PATCH 05/15] Try skipping coverage for pull requests.

---
 Makefile | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index cc5308fe1f..fd70f0b3d9 100644
--- a/Makefile
+++ b/Makefile
@@ -49,8 +49,18 @@ PIP_OPTIONS := $(ST2_PIP_OPTIONS)
 
 NOSE_OPTS := --rednose --immediate --with-parallel
 NOSE_TIME := $(NOSE_TIME)
-NOSE_COVERAGE_FLAGS := --with-coverage --cover-branches --cover-erase
-NOSE_COVERAGE_PACKAGES := --cover-package=$(COMPONENTS_TEST_COMMA)
+
+TRAVIS_PULL_REQUEST := $(TRAVIS_PULL_REQUEST)
+
+# NOTE: We only run coverage on master and version branches and not on pull requests since
+# it has a big performance overhead and is very slow.
+ifdef TRAVIS_PULL_REQUEST
+	NOSE_COVERAGE_FLAGS := --with-coverage --cover-branches --cover-erase
+	NOSE_COVERAGE_PACKAGES := --cover-package=$(COMPONENTS_TEST_COMMA)
+else
+	NOSE_COVERAGE_FLAGS = ""
+	NOSE_COVERAGE_PACKAGES = ""
+endif
 
 ifdef NOSE_TIME
 	NOSE_OPTS := --rednose --immediate --with-parallel --with-timer

From 61421f3a765152fd3da7ad45874d40e7de5748e8 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Tue, 24 Jul 2018 18:03:33 +0200
Subject: [PATCH 06/15] Only run coverage on master branch and not on pull
 requests.

---
 Makefile | 42 +++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/Makefile b/Makefile
index fd70f0b3d9..586c05d996 100644
--- a/Makefile
+++ b/Makefile
@@ -50,18 +50,6 @@ PIP_OPTIONS := $(ST2_PIP_OPTIONS)
 NOSE_OPTS := --rednose --immediate --with-parallel
 NOSE_TIME := $(NOSE_TIME)
 
-TRAVIS_PULL_REQUEST := $(TRAVIS_PULL_REQUEST)
-
-# NOTE: We only run coverage on master and version branches and not on pull requests since
-# it has a big performance overhead and is very slow.
-ifdef TRAVIS_PULL_REQUEST
-	NOSE_COVERAGE_FLAGS := --with-coverage --cover-branches --cover-erase
-	NOSE_COVERAGE_PACKAGES := --cover-package=$(COMPONENTS_TEST_COMMA)
-else
-	NOSE_COVERAGE_FLAGS = ""
-	NOSE_COVERAGE_PACKAGES = ""
-endif
-
 ifdef NOSE_TIME
 	NOSE_OPTS := --rednose --immediate --with-parallel --with-timer
 endif
@@ -70,9 +58,18 @@ ifndef PIP_OPTIONS
 	PIP_OPTIONS :=
 endif
 
-ifneq ($(INCLUDE_TESTS_IN_COVERAGE),)
-	NOSE_COVERAGE_FLAGS += --cover-tests
-	NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),$(COMPONENTS_TEST_MODULES_COMMA)
+# NOTE: We only run coverage on master and version branches and not on pull requests since
+# it has a big performance overhead and is very slow.
+TRAVIS_PULL_REQUEST := $(TRAVIS_PULL_REQUEST)
+
+ifeq ($(TRAVIS_PULL_REQUEST),false)
+	NOSE_COVERAGE_FLAGS := --with-coverage --cover-branches --cover-erase
+	NOSE_COVERAGE_PACKAGES := --cover-package=$(COMPONENTS_TEST_COMMA)
+
+	ifneq ($(INCLUDE_TESTS_IN_COVERAGE),)
+		NOSE_COVERAGE_FLAGS += --cover-tests
+		NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),$(COMPONENTS_TEST_MODULES_COMMA)
+	endif
 endif
 
 .PHONY: all
@@ -435,7 +432,9 @@ unit-tests: requirements .unit-tests
 
 .PHONY: .run-unit-tests-coverage
 ifneq ($(INCLUDE_TESTS_IN_COVERAGE),)
-.run-unit-tests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),tests.unit
+	ifneq ($(NOSE_COVERAGE_FLAGS),)
+		.run-unit-tests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),tests.unit
+	endif
 endif
 .run-unit-tests-coverage:
 	@echo
@@ -448,10 +447,9 @@ endif
 		echo "Running tests in" $$component; \
 		echo "-----------------------------------------------------------"; \
 		. $(VIRTUALENV_DIR)/bin/activate; \
-		    COVERAGE_FILE=.coverage.unit.$$(echo $$component | tr '/' '.') \
-		    nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \
-		    $(NOSE_COVERAGE_PACKAGES) \
-		    $$component/tests/unit || exit 1; \
+			COVERAGE_FILE=.coverage.unit.$$(echo $$component | tr '/' '.') \
+			nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) \
+			$$component/tests/unit || exit 1; \
 		echo "-----------------------------------------------------------"; \
 		echo "Done running tests in" $$component; \
 		echo "==========================================================="; \
@@ -507,7 +505,9 @@ itests: requirements .itests
 
 .PHONY: .run-integration-tests-coverage
 ifneq ($(INCLUDE_TESTS_IN_COVERAGE),)
-.run-integration-tests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),tests.integration
+	ifneq ($(NOSE_COVERAGE_FLAGS),)
+		.run-integration-tests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),tests.integration
+	endif
 endif
 .run-integration-tests-coverage:
 	@echo

From a97bb04d89bdf1113ac1affbfde14016ac46aac5 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Tue, 24 Jul 2018 18:29:22 +0200
Subject: [PATCH 07/15] Only submit coverage report on non pull request
 branches.

---
 .travis.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 0d79960b38..e1d8a35214 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -82,7 +82,8 @@ before_cache:
   - if [ ${TRAVIS_PULL_REQUEST} = 'false' ]; then rm -rf virtualenv/; fi
 
 after_success:
-  - if [ ${TASK} = 'ci-unit' ] || [ ${TASK} = 'ci-integration' ]; then codecov; fi
+  # NOTE: We only generate and submit coverage report for master and version branches
+  - if [ ${TASK} = 'ci-unit' ] || [ ${TASK} = 'ci-integration' ] && [ ${TRAVIS_PULL_REQUEST} = 'false' ]; then codecov; fi
 
 # https://docs.travis-ci.com/user/notifications/#Webhooks-Delivery-Format
 #notifications:

From fc49d30d29e24ef9b49ba534215a643b0a3ee1c0 Mon Sep 17 00:00:00 2001
From: blag <blag@users.noreply.github.com>
Date: Tue, 24 Jul 2018 11:00:05 -0700
Subject: [PATCH 08/15] Clean up new makefile logic

---
 Makefile | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/Makefile b/Makefile
index 586c05d996..0a13aba79a 100644
--- a/Makefile
+++ b/Makefile
@@ -60,8 +60,6 @@ endif
 
 # NOTE: We only run coverage on master and version branches and not on pull requests since
 # it has a big performance overhead and is very slow.
-TRAVIS_PULL_REQUEST := $(TRAVIS_PULL_REQUEST)
-
 ifeq ($(TRAVIS_PULL_REQUEST),false)
 	NOSE_COVERAGE_FLAGS := --with-coverage --cover-branches --cover-erase
 	NOSE_COVERAGE_PACKAGES := --cover-package=$(COMPONENTS_TEST_COMMA)
@@ -70,6 +68,10 @@ ifeq ($(TRAVIS_PULL_REQUEST),false)
 		NOSE_COVERAGE_FLAGS += --cover-tests
 		NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),$(COMPONENTS_TEST_MODULES_COMMA)
 	endif
+else
+	# If we aren't running test coverage, don't try to include tests in coverage
+	# results
+	INCLUDE_TESTS_IN_COVERAGE :=
 endif
 
 .PHONY: all
@@ -432,9 +434,7 @@ unit-tests: requirements .unit-tests
 
 .PHONY: .run-unit-tests-coverage
 ifneq ($(INCLUDE_TESTS_IN_COVERAGE),)
-	ifneq ($(NOSE_COVERAGE_FLAGS),)
-		.run-unit-tests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),tests.unit
-	endif
+.run-unit-tests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),tests.unit
 endif
 .run-unit-tests-coverage:
 	@echo
@@ -447,9 +447,10 @@ endif
 		echo "Running tests in" $$component; \
 		echo "-----------------------------------------------------------"; \
 		. $(VIRTUALENV_DIR)/bin/activate; \
-			COVERAGE_FILE=.coverage.unit.$$(echo $$component | tr '/' '.') \
-			nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) \
-			$$component/tests/unit || exit 1; \
+		    COVERAGE_FILE=.coverage.unit.$$(echo $$component | tr '/' '.') \
+		    nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \
+		    $(NOSE_COVERAGE_PACKAGES) \
+		    $$component/tests/unit || exit 1; \
 		echo "-----------------------------------------------------------"; \
 		echo "Done running tests in" $$component; \
 		echo "==========================================================="; \
@@ -505,9 +506,7 @@ itests: requirements .itests
 
 .PHONY: .run-integration-tests-coverage
 ifneq ($(INCLUDE_TESTS_IN_COVERAGE),)
-	ifneq ($(NOSE_COVERAGE_FLAGS),)
-		.run-integration-tests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),tests.integration
-	endif
+.run-integration-tests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),tests.integration
 endif
 .run-integration-tests-coverage:
 	@echo

From 533966e905d6b3ef16af4061a4a1b1df3b0a112d Mon Sep 17 00:00:00 2001
From: blag <blag@users.noreply.github.com>
Date: Tue, 24 Jul 2018 12:09:11 -0700
Subject: [PATCH 09/15] Use and ignore virtualenv on OS X

---
 Makefile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 0a13aba79a..bef5553e66 100644
--- a/Makefile
+++ b/Makefile
@@ -191,7 +191,7 @@ generate-api-spec: requirements .generate-api-spec
 	echo "# Edit st2common/st2common/openapi.yaml.j2 and then run" >> st2common/st2common/openapi.yaml
 	echo "# make .generate-api-spec" >> st2common/st2common/openapi.yaml
 	echo "# to generate the final spec file" >> st2common/st2common/openapi.yaml
-	. virtualenv/bin/activate; st2common/bin/st2-generate-api-spec --config-file conf/st2.dev.conf >> st2common/st2common/openapi.yaml
+	. $(VIRTUALENV_DIR)/bin/activate; st2common/bin/st2-generate-api-spec --config-file conf/st2.dev.conf >> st2common/st2common/openapi.yaml
 
 .PHONY: circle-lint-api-spec
 circle-lint-api-spec:
@@ -240,13 +240,13 @@ clean: .cleanpycs
 compile:
 	@echo "======================= compile ========================"
 	@echo "------- Compile all .py files (syntax check test - Python 2) ------"
-	@if python -c 'import compileall,re; compileall.compile_dir(".", rx=re.compile(r"/virtualenv|.tox"), quiet=True)' | grep .; then exit 1; else exit 0; fi
+	@if python -c 'import compileall,re; compileall.compile_dir(".", rx=re.compile(r"/virtualenv|virtualenv-osx|.tox"), quiet=True)' | grep .; then exit 1; else exit 0; fi
 
 .PHONY: compilepy3
 compilepy3:
 	@echo "======================= compile ========================"
 	@echo "------- Compile all .py files (syntax check test - Python 3) ------"
-	@if python3 -c 'import compileall,re; compileall.compile_dir(".", rx=re.compile(r"/virtualenv|.tox|./st2tests/st2tests/fixtures/packs/test"), quiet=True)' | grep .; then exit 1; else exit 0; fi
+	@if python3 -c 'import compileall,re; compileall.compile_dir(".", rx=re.compile(r"/virtualenv|virtualenv-osx|.tox|./st2tests/st2tests/fixtures/packs/test"), quiet=True)' | grep .; then exit 1; else exit 0; fi
 
 .PHONY: .cleanpycs
 .cleanpycs:
@@ -325,14 +325,14 @@ requirements: virtualenv .sdist-requirements
 	$(VIRTUALENV_DIR)/bin/pip install --upgrade "virtualenv==15.1.0" # Required for packs.install in dev envs.
 
 	# Generate all requirements to support current CI pipeline.
-	$(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv -s st2*/in-requirements.txt contrib/runners/*/in-requirements.txt -f fixed-requirements.txt -o requirements.txt
+	$(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv,virtualenv-osx -s st2*/in-requirements.txt contrib/runners/*/in-requirements.txt -f fixed-requirements.txt -o requirements.txt
 
 	# Generate finall requirements.txt file for each component
 	@for component in $(COMPONENTS_WITH_RUNNERS); do\
 		echo "==========================================================="; \
 		echo "Generating requirements.txt for" $$component; \
 		echo "==========================================================="; \
-		$(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv -s $$component/in-requirements.txt -f fixed-requirements.txt -o $$component/requirements.txt; \
+		$(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv,virtualenv-osx -s $$component/in-requirements.txt -f fixed-requirements.txt -o $$component/requirements.txt; \
 	done
 
 	# Fix for Travis CI race

From 92366e5627c42d4fd59d64360f65df180a507d8f Mon Sep 17 00:00:00 2001
From: blag <blag@users.noreply.github.com>
Date: Tue, 24 Jul 2018 12:10:01 -0700
Subject: [PATCH 10/15] Don't try to combine coverage data if we aren't running
 tests with coverage instrumentation

---
 Makefile | 54 ++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 34 insertions(+), 20 deletions(-)

diff --git a/Makefile b/Makefile
index bef5553e66..75eb48ba74 100644
--- a/Makefile
+++ b/Makefile
@@ -458,29 +458,36 @@ endif
 
 .PHONY: .combine-unit-tests-coverage
 .combine-unit-tests-coverage: .run-unit-tests-coverage
-	. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.unit \
-	    coverage combine .coverage.unit.*
+	@if [ -n "$(NOSE_COVERAGE_FLAGS)" ]; then \
+	    . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.unit \
+	        coverage combine .coverage.unit.*; \
+	fi
 
 .coverage.unit:
-	@compgen -G '.coverage.unit.*' && \
+	@if compgen -G '.coverage.unit.*'; then \
 		for coverage_result in $$(compgen -G '.coverage.unit.*'); do \
 			echo "Combining data from $${coverage_result}"; \
 			. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.unit \
 			coverage combine $${coverage_result}; \
-		done \
-	|| \
+		done; \
+	else \
 		echo "Running unit tests"; \
-		make .combine-unit-tests-coverage
+		make .combine-unit-tests-coverage; \
+	fi
 
 .PHONY: .report-unit-tests-coverage
 .report-unit-tests-coverage: .coverage.unit
-	. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.unit \
-	    coverage report
+	@if [ -n "$(NOSE_COVERAGE_FLAGS)" ]; then \
+	    . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.unit \
+	        coverage report; \
+	fi
 
 .PHONY: .unit-tests-coverage-html
 .unit-tests-coverage-html: .coverage.unit
-	. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.unit \
-	    coverage html
+	@if [ -n "$(NOSE_COVERAGE_FLAGS)" ]; then \
+	    . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.unit \
+	        coverage html; \
+	fi
 
 .PHONY: itests
 itests: requirements .itests
@@ -530,29 +537,36 @@ endif
 
 .PHONY: .combine-integration-tests-coverage
 .combine-integration-tests-coverage: .run-integration-tests-coverage
-	. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.integration \
-	    coverage combine .coverage.integration.*
+	@if [ -n "$(NOSE_COVERAGE_FLAGS)" ]; then \
+	    . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.integration \
+	        coverage combine .coverage.integration.*; \
+	fi
 
 .coverage.integration:
-	@compgen -G '.coverage.integration.*' && \
+	@if compgen -G '.coverage.integration.*'; then \
 		for coverage_result in $$(compgen -G '.coverage.integration.*'); do \
 			echo "Combining data from $${coverage_result}"; \
 			. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.integration \
 			coverage combine $${coverage_result}; \
-		done \
-	|| \
+		done; \
+	else \
 		echo "Running integration tests"; \
-		make .combine-integration-tests-coverage
+		make .combine-integration-tests-coverage; \
+	fi
 
 .PHONY: .report-integration-tests-coverage
 .report-integration-tests-coverage: .coverage.integration
-	@. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.integration \
-	    coverage report
+	@if [ -n "$(NOSE_COVERAGE_FLAGS)" ]; then \
+	    . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.integration \
+	        coverage report; \
+	fi
 
 .PHONY: .integration-tests-coverage-html
 .integration-tests-coverage-html: .coverage.integration
-	@. $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.integration \
-	    coverage html
+	@if [ -n "$(NOSE_COVERAGE_FLAGS)" ]; then \
+	    . $(VIRTUALENV_DIR)/bin/activate; COVERAGE_FILE=.coverage.integration \
+	        coverage html; \
+	fi
 
 .PHONY: .itests-coverage-html
 .itests-coverage-html: .integration-tests-coverage-html

From 0556bb7b3921df29c4023d0f46056bfd47348cf9 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Tue, 24 Jul 2018 23:54:54 +0200
Subject: [PATCH 11/15] Use -e when installing orchestra dependency from git
 otherwise the dependency won't be updated locally when running "make
 requirements" and either virtualenv needs to be re-created for new version to
 be installed or pip uninstall orchestra ran before make requirements.

---
 requirements.txt              | 2 +-
 st2common/in-requirements.txt | 2 +-
 st2common/requirements.txt    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index d66b17f085..ec6e5cea7c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,5 @@
 # Don't edit this file. It's generated automatically!
+-e git+https://github.com/StackStorm/orchestra.git@master#egg=orchestra
 RandomWords
 apscheduler==3.5.1
 argcomplete
@@ -7,7 +8,6 @@ cryptography==2.2.2
 eventlet==0.23.0
 flex==6.13.1
 git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper
-git+https://github.com/StackStorm/orchestra.git#egg=orchestra
 git+https://github.com/StackStorm/python-mistralclient.git#egg=python-mistralclient
 git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file
 gitpython==2.1.10
diff --git a/st2common/in-requirements.txt b/st2common/in-requirements.txt
index 9863d4295b..0ea8b152ac 100644
--- a/st2common/in-requirements.txt
+++ b/st2common/in-requirements.txt
@@ -9,7 +9,7 @@ jsonschema
 kombu
 mongoengine
 networkx
-git+https://github.com/StackStorm/orchestra.git#egg=orchestra
+-e git+https://github.com/StackStorm/orchestra.git@master#egg=orchestra
 oslo.config
 paramiko
 pyyaml
diff --git a/st2common/requirements.txt b/st2common/requirements.txt
index 5e1b6d5b58..2375597441 100644
--- a/st2common/requirements.txt
+++ b/st2common/requirements.txt
@@ -1,9 +1,9 @@
 # Don't edit this file. It's generated automatically!
+-e git+https://github.com/StackStorm/orchestra.git@master#egg=orchestra
 apscheduler==3.5.1
 cryptography==2.2.2
 eventlet==0.23.0
 flex==6.13.1
-git+https://github.com/StackStorm/orchestra.git#egg=orchestra
 greenlet==0.4.13
 ipaddr
 jinja2

From 60d85e3390b16289c888a60bfa017d5826eb500f Mon Sep 17 00:00:00 2001
From: blag <blag@users.noreply.github.com>
Date: Tue, 24 Jul 2018 16:03:13 -0700
Subject: [PATCH 12/15] Refactor to separate TRAVIS_PULL_REQUEST from
 ENABLE_COVERAGE

---
 Makefile | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/Makefile b/Makefile
index 75eb48ba74..f286cf40b5 100644
--- a/Makefile
+++ b/Makefile
@@ -61,19 +61,23 @@ endif
 # NOTE: We only run coverage on master and version branches and not on pull requests since
 # it has a big performance overhead and is very slow.
 ifeq ($(TRAVIS_PULL_REQUEST),false)
+	ENABLE_COVERAGE := yes
+endif
+
+ifdef ENABLE_COVERAGE
 	NOSE_COVERAGE_FLAGS := --with-coverage --cover-branches --cover-erase
 	NOSE_COVERAGE_PACKAGES := --cover-package=$(COMPONENTS_TEST_COMMA)
-
-	ifneq ($(INCLUDE_TESTS_IN_COVERAGE),)
-		NOSE_COVERAGE_FLAGS += --cover-tests
-		NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),$(COMPONENTS_TEST_MODULES_COMMA)
-	endif
 else
-	# If we aren't running test coverage, don't try to include tests in coverage
-	# results
 	INCLUDE_TESTS_IN_COVERAGE :=
 endif
 
+# If we aren't running test coverage, don't try to include tests in coverage
+# results
+ifdef INCLUDE_TESTS_IN_COVERAGE
+	NOSE_COVERAGE_FLAGS += --cover-tests
+	NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),$(COMPONENTS_TEST_MODULES_COMMA)
+endif
+
 .PHONY: all
 all: requirements configgen check tests
 
@@ -108,10 +112,16 @@ play:
 	@echo
 	@echo COMPONENT_PYTHONPATH=$(COMPONENT_PYTHONPATH)
 	@echo
+	@echo TRAVIS_PULL_REQUEST=$(TRAVIS_PULL_REQUEST)
+	@echo
+	@echo ENABLE_COVERAGE=$(ENABLE_COVERAGE)
+	@echo
 	@echo NOSE_COVERAGE_FLAGS=$(NOSE_COVERAGE_FLAGS)
 	@echo
 	@echo NOSE_COVERAGE_PACKAGES=$(NOSE_COVERAGE_PACKAGES)
 	@echo
+	@echo INCLUDE_TESTS_IN_COVERAGE=$(INCLUDE_TESTS_IN_COVERAGE)
+	@echo
 
 .PHONY: check
 check: requirements flake8 checklogs
@@ -433,7 +443,7 @@ unit-tests: requirements .unit-tests
 	done
 
 .PHONY: .run-unit-tests-coverage
-ifneq ($(INCLUDE_TESTS_IN_COVERAGE),)
+ifdef INCLUDE_TESTS_IN_COVERAGE
 .run-unit-tests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),tests.unit
 endif
 .run-unit-tests-coverage:
@@ -512,7 +522,7 @@ itests: requirements .itests
 	done
 
 .PHONY: .run-integration-tests-coverage
-ifneq ($(INCLUDE_TESTS_IN_COVERAGE),)
+ifdef INCLUDE_TESTS_IN_COVERAGE
 .run-integration-tests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),tests.integration
 endif
 .run-integration-tests-coverage:
@@ -583,7 +593,7 @@ mistral-itests: requirements .mistral-itests
 	. $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v st2tests/integration/mistral || exit 1;
 
 .PHONY: .run-mistral-itests-coverage
-ifneq ($(INCLUDE_TESTS_IN_COVERAGE),)
+ifdef INCLUDE_TESTS_IN_COVERAGE
 .run-mistral-itests-coverage: NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),st2tests.mistral.integration
 endif
 .run-mistral-itests-coverage:

From 52543aa7cc4671e568ed8dcb449b5ebf2b0470aa Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Wed, 25 Jul 2018 09:58:25 +0200
Subject: [PATCH 13/15] Dummy commit to force the build.

---
 .travis.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index e1d8a35214..6c2d7c8609 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -84,10 +84,3 @@ before_cache:
 after_success:
   # NOTE: We only generate and submit coverage report for master and version branches
   - if [ ${TASK} = 'ci-unit' ] || [ ${TASK} = 'ci-integration' ] && [ ${TRAVIS_PULL_REQUEST} = 'false' ]; then codecov; fi
-
-# https://docs.travis-ci.com/user/notifications/#Webhooks-Delivery-Format
-#notifications:
-#  webhooks:
-#    #- https://ci-webhooks.stackstorm.net/webhooks/build/events
-#    # See: webhook.site/#/06fde88c-1610-4c85-ba4e-d9bcacfefe4c
-#    - http://webhook.site/06fde88c-1610-4c85-ba4e-d9bcacfefe4c

From 3991bc1a9335daa8033165052f59a221d5dbbb7c Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Wed, 25 Jul 2018 10:05:31 +0200
Subject: [PATCH 14/15] For a more user-friendly matrix display, give Travis
 jobs a name.

---
 .travis.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 6c2d7c8609..f684861110 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,16 +16,22 @@ matrix:
   include:
     - env: TASK=ci-unit NODE_INDEX=0 NODE_TOTAL=2
       python: 2.7
+      name: "Unit Tests (Python 2.7) - 1"
     - env: TASK=ci-unit NODE_INDEX=1 NODE_TOTAL=2
       python: 2.7
+      name: "Unit Tests (Python 2.7) - 2"
     - env: TASK=ci-integration
       python: 2.7
+      name: "Integration Tests (Python 2.7)"
     - env: TASK="ci-checks ci-packs-tests"
       python: 2.7
+      name: "Lint Checks, Packs Tests (Python 2.7)"
     - env: TASK="compilepy3 ci-py3-unit" CACHE_NAME=py3
       python: 3.6
+      name: "Unit Tests (Python 3.6)"
     - env: TASK="ci-py3-integration" CACHE_NAME=py3
       python: 3.6
+      name: "Integration Tests (Python 3.6)"
 addons:
   apt:
     sources:

From 6a09130941da67d6342a3e8e7d8e28128716a160 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Wed, 25 Jul 2018 10:31:36 +0200
Subject: [PATCH 15/15] Remove -e flag since it some how brakes Circle CI
 build.

---
 requirements.txt              | 2 +-
 st2common/in-requirements.txt | 2 +-
 st2common/requirements.txt    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index ec6e5cea7c..5590389b6a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,4 @@
 # Don't edit this file. It's generated automatically!
--e git+https://github.com/StackStorm/orchestra.git@master#egg=orchestra
 RandomWords
 apscheduler==3.5.1
 argcomplete
@@ -8,6 +7,7 @@ cryptography==2.2.2
 eventlet==0.23.0
 flex==6.13.1
 git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper
+git+https://github.com/StackStorm/orchestra.git@master#egg=orchestra
 git+https://github.com/StackStorm/python-mistralclient.git#egg=python-mistralclient
 git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file
 gitpython==2.1.10
diff --git a/st2common/in-requirements.txt b/st2common/in-requirements.txt
index 0ea8b152ac..e69ce3a558 100644
--- a/st2common/in-requirements.txt
+++ b/st2common/in-requirements.txt
@@ -9,7 +9,7 @@ jsonschema
 kombu
 mongoengine
 networkx
--e git+https://github.com/StackStorm/orchestra.git@master#egg=orchestra
+git+https://github.com/StackStorm/orchestra.git@master#egg=orchestra
 oslo.config
 paramiko
 pyyaml
diff --git a/st2common/requirements.txt b/st2common/requirements.txt
index 2375597441..a483376690 100644
--- a/st2common/requirements.txt
+++ b/st2common/requirements.txt
@@ -1,9 +1,9 @@
 # Don't edit this file. It's generated automatically!
--e git+https://github.com/StackStorm/orchestra.git@master#egg=orchestra
 apscheduler==3.5.1
 cryptography==2.2.2
 eventlet==0.23.0
 flex==6.13.1
+git+https://github.com/StackStorm/orchestra.git@master#egg=orchestra
 greenlet==0.4.13
 ipaddr
 jinja2