From 8730a849c5393578b2fca16c7ac3b1c40908b9ef Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Fri, 26 Mar 2021 23:48:47 +0100
Subject: [PATCH 01/29] Test running unit tests in multiple jobs in parallel.

---
 .github/workflows/ci.yaml | 39 +++++++++++++++++++++++++++++++++++++--
 Makefile                  |  5 +++++
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index cfaeff6b8e..53f43a7417 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -47,23 +47,55 @@ jobs:
         include:
           - name: 'Lint Checks'
             task: 'ci-checks'
+            nosetests_node_total: 1
+            nosetests_node_index: 0
             python-version: '3.6'
           - name: 'Compile'
             task: 'ci-compile'
+            nosetests_node_total: 1
+            nosetests_node_index: 0
             python-version: '3.6'
           - name: 'Pack Tests'
             task: 'ci-packs-tests'
+            nosetests_node_total: 1
+            nosetests_node_index: 0
             python-version: '3.6'
-          - name: 'Unit Tests'
+          - name: 'Unit Tests (chunk 1)'
             task: 'ci-unit'
+            nosetests_node_total: 3
+            nosetests_node_index: 0
             python-version: '3.6'
-          - name: 'Integration Tests'
+          - name: 'Unit Tests (chunk 2)'
+            task: 'ci-unit'
+            nosetests_node_total: 3
+            nosetests_node_index: 1
+            python-version: '3.6'
+          - name: 'Unit Tests (chunk 3)'
+            task: 'ci-unit'
+            nosetests_node_total: 3
+            nosetests_node_index: 2
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 1)'
             task: 'ci-integration'
+            nosetests_node_total: 3
+            nosetests_node_index: 0
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 2)'
+            task: 'ci-integration'
+            nosetests_node_total: 3
+            nosetests_node_index: 1
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 3)'
+            task: 'ci-integration'
+            nosetests_node_total: 3
+            nosetests_node_index: 2
             python-version: '3.6'
           # This job is slow so we only run in on a daily basis
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
           #   python-version: '3.6'
+          #   nosetests_node_total: 1
+          #  nosetests_node_ index: 0
     services:
       mongo:
         image: mongo:4.0
@@ -120,6 +152,9 @@ jobs:
     env:
       TASK: '${{ matrix.task }}'
 
+      NODE_TOTAL: '${{ matrix.nosetests_node_total }}'
+      NODE_INDEX: '${{ matrix.nosetests_node_index }}'
+
       # We need to explicitly specify terminal width otherwise some CLI tests fail on container
       # environments where small terminal size is used.
       COLUMNS: '120'
diff --git a/Makefile b/Makefile
index 6728f2ac68..d5f0ce3319 100644
--- a/Makefile
+++ b/Makefile
@@ -155,6 +155,11 @@ play:
 	@echo
 	@echo INCLUDE_TESTS_IN_COVERAGE=$(INCLUDE_TESTS_IN_COVERAGE)
 	@echo
+	@echo NODE_TOTAL=$(NODE_TOTAL)
+	@echo
+	@echo
+	@echo NODE_INDEX=$(NODE_INDEX)
+	@echo
 
 .PHONY: check
 check: check-requirements check-sdist-requirements flake8 checklogs

From 78baa3b4286e27dfa1c829601c1befdfc8eea751 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 15:46:32 +0100
Subject: [PATCH 02/29] Fix test case and make sure we don't realy on state
 created by another test.

This way test method can run in a standalone fashion.
---
 .../test_register_content_script.py           | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/st2common/tests/integration/test_register_content_script.py b/st2common/tests/integration/test_register_content_script.py
index 0a0dc8e7f1..24f1712083 100644
--- a/st2common/tests/integration/test_register_content_script.py
+++ b/st2common/tests/integration/test_register_content_script.py
@@ -171,17 +171,32 @@ def test_register_setup_virtualenvs(self):
         self.assertEqual(exit_code, 0)
 
     def test_register_recreate_virtualenvs(self):
-        # Single pack
+        # 1. Register the pack and ensure it exists and doesn't rely on state from previous
+        # test methods
         pack_dir = os.path.join(get_fixtures_packs_base_path(), "dummy_pack_1")
 
         cmd = BASE_CMD_ARGS + [
             "--register-pack=%s" % (pack_dir),
-            "--register-recreate-virtualenvs",
+            "--register-setup-virtualenvs",
             "--register-no-fail-on-failure",
         ]
         exit_code, stdout, stderr = run_command(cmd=cmd)
 
         self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
         self.assertIn("Setup virtualenv for 1 pack(s)", stderr)
+        self.assertEqual(exit_code, 0)
+
+        # 2. Run it again with --register-recreate-virtualenvs flag
+        pack_dir = os.path.join(get_fixtures_packs_base_path(), "dummy_pack_1")
+
+        cmd = BASE_CMD_ARGS + [
+            "--register-pack=%s" % (pack_dir),
+            "--register-recreate-virtualenvs",
+            "--register-no-fail-on-failure",
+        ]
+        exit_code, stdout, stderr = run_command(cmd=cmd)
+
+        self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
         self.assertIn("Virtualenv successfully removed.", stderr)
+        self.assertIn("Setup virtualenv for 1 pack(s)", stderr)
         self.assertEqual(exit_code, 0)

From 6287ddd9de6fbc031705b9a3eb697789e2e90e50 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 16:20:24 +0100
Subject: [PATCH 03/29] Add additional logging.

---
 .../tests/integration/test_sensor_container.py      | 13 +++++++------
 st2tests/st2tests/base.py                           |  1 +
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/st2reactor/tests/integration/test_sensor_container.py b/st2reactor/tests/integration/test_sensor_container.py
index 41eb3307bc..687c9f9102 100644
--- a/st2reactor/tests/integration/test_sensor_container.py
+++ b/st2reactor/tests/integration/test_sensor_container.py
@@ -111,7 +111,7 @@ def test_child_processes_are_killed_on_sigint(self):
         process = self._start_sensor_container()
 
         # Give it some time to start up
-        concurrency.sleep(5)
+        concurrency.sleep(7)
 
         # Assert process has started and is running
         self.assertProcessIsRunning(process=process)
@@ -139,7 +139,7 @@ def test_child_processes_are_killed_on_sigterm(self):
         process = self._start_sensor_container()
 
         # Give it some time to start up
-        concurrency.sleep(3)
+        concurrency.sleep(5)
 
         # Verify container process and children sensor / wrapper processes are running
         pp = psutil.Process(process.pid)
@@ -152,7 +152,7 @@ def test_child_processes_are_killed_on_sigterm(self):
 
         # SIGTERM causes graceful shutdown so give it some time to gracefuly shut down the sensor
         # child processes
-        concurrency.sleep(PROCESS_EXIT_TIMEOUT + 5)
+        concurrency.sleep(PROCESS_EXIT_TIMEOUT + 8)
 
         # Verify parent and children processes have exited
         self.assertProcessExited(proc=pp)
@@ -164,7 +164,7 @@ def test_child_processes_are_killed_on_sigkill(self):
         process = self._start_sensor_container()
 
         # Give it some time to start up
-        concurrency.sleep(4)
+        concurrency.sleep(5)
 
         # Verify container process and children sensor / wrapper processes are running
         pp = psutil.Process(process.pid)
@@ -198,7 +198,7 @@ def test_single_sensor_mode(self):
         pp = psutil.Process(process.pid)
 
         # Give it some time to start up
-        concurrency.sleep(4)
+        concurrency.sleep(5)
 
         stdout = process.stdout.read()
         self.assertTrue(
@@ -224,7 +224,7 @@ def test_single_sensor_mode(self):
         pp = psutil.Process(process.pid)
 
         # Give it some time to start up
-        concurrency.sleep(8)
+        concurrency.sleep(1)
 
         # Container should exit and not respawn a sensor in single sensor mode
         stdout = process.stdout.read()
@@ -242,6 +242,7 @@ def test_single_sensor_mode(self):
 
     def _start_sensor_container(self, cmd=DEFAULT_CMD):
         subprocess = concurrency.get_subprocess_module()
+        print("Using command: %s" % (" ".join(cmd)))
         process = subprocess.Popen(
             cmd,
             stdout=subprocess.PIPE,
diff --git a/st2tests/st2tests/base.py b/st2tests/st2tests/base.py
index 4a4964763d..de90571684 100644
--- a/st2tests/st2tests/base.py
+++ b/st2tests/st2tests/base.py
@@ -575,6 +575,7 @@ def tearDown(self):
                 except:
                     stderr = None
 
+                print("Stopping process with pid %s" % (process.id))
                 print('Process "%s"' % (process.pid))
                 print("Stdout: %s" % (stdout))
                 print("Stderr: %s" % (stderr))

From bffff5eaed893a90df5c654068cd7d2dc5bd016b Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 16:38:19 +0100
Subject: [PATCH 04/29] Add additional parallel job.

---
 .github/workflows/ci.yaml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 53f43a7417..e7dcf603c2 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -77,19 +77,24 @@ jobs:
             python-version: '3.6'
           - name: 'Integration Tests (chunk 1)'
             task: 'ci-integration'
-            nosetests_node_total: 3
+            nosetests_node_total: 4
             nosetests_node_index: 0
             python-version: '3.6'
           - name: 'Integration Tests (chunk 2)'
             task: 'ci-integration'
-            nosetests_node_total: 3
+            nosetests_node_total: 4
             nosetests_node_index: 1
             python-version: '3.6'
           - name: 'Integration Tests (chunk 3)'
             task: 'ci-integration'
-            nosetests_node_total: 3
+            nosetests_node_total: 4
             nosetests_node_index: 2
             python-version: '3.6'
+          - name: 'Integration Tests (chunk 4)'
+            task: 'ci-integration'
+            nosetests_node_total: 4
+            nosetests_node_index: 3
+            python-version: '3.6'
           # This job is slow so we only run in on a daily basis
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'

From 6de8b11f80ab0a19f9ce740271c25e9996dcfdd6 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 19:16:36 +0100
Subject: [PATCH 05/29] Only try running redis container for integration tests
 where it's needed.

---
 .github/workflows/ci.yaml | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index e7dcf603c2..f6ff0fe27f 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -141,18 +141,20 @@ jobs:
       # Used for the coordination backend for integration tests
       # TODO: Only start this for integration tests via job step
       # https://github.community/t/conditional-services-in-a-job/135301/3
-      redis:
-        # Docker Hub image
-        image: redis
-        # Set health checks to wait until redis has started
-        options: >-
-          --name "redis"
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 6379:6379/tcp
+      # NOTE: To speed things up, we only start redis for integration tests
+      # where it's needed
+      # redis:
+      #   # Docker Hub image
+      #   image: redis
+      #   # Set health checks to wait until redis has started
+      #   options: >-
+      #     --name "redis"
+      #     --health-cmd "redis-cli ping"
+      #     --health-interval 10s
+      #     --health-timeout 5s
+      #     --health-retries 5
+      #   ports:
+      #     - 6379:6379/tcp
 
     env:
       TASK: '${{ matrix.task }}'
@@ -273,6 +275,12 @@ jobs:
           cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" "${ST2_CONF}"
           ./scripts/ci/add-itest-user-key.sh
           sudo .circle/add-itest-user.sh
+      - name: Run Redis Service Container
+        if: "${{ env.TASK == 'ci-integration' }}"
+        timeout-minutes: 2
+        run: |
+          docker run --rm --detach -p 127.0.0.1:6379:6379/tcp --name redis redis:latest
+          until [ "$(docker inspect -f {{.State.Running}} redis)" == "true" ]; do sleep 0.1; done
       - name: Permissions Workaround
         if: "${{ env.TASK == 'ci-packs-tests' || env.TASK == 'ci-integration' }}"
         run: |
@@ -310,6 +318,9 @@ jobs:
         if: "${{ success() && ((env.TASK == 'ci-unit') || (env.TASK == 'ci-integration')) && (env.ENABLE_COVERAGE == 'yes') }}"
         run: |
           ./scripts/ci/submit-codecov-coverage.sh
+      - name: Stop Redis Service Container
+        if: "${{ always() && env.TASK == 'ci-integration' }}"
+        run: docker stop redis || true
   slack-notification:
     name: Slack notification for failed master builds
     if: always()

From 6b3c1e45b3cbe95fd21e9630e4ae27908bfc80b7 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 19:20:53 +0100
Subject: [PATCH 06/29] Move redis to test requirements.

---
 scripts/github/prepare-integration.sh | 3 ---
 test-requirements.txt                 | 2 ++
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/github/prepare-integration.sh b/scripts/github/prepare-integration.sh
index 35e138c8ea..9e13ade526 100755
--- a/scripts/github/prepare-integration.sh
+++ b/scripts/github/prepare-integration.sh
@@ -26,9 +26,6 @@ echo ""
 cat conf/st2.ci.conf || true
 echo ""
 
-# Needed by the coordination backend
-pip install "redis==3.5.3"
-
 # install st2 client
 python ./st2client/setup.py develop
 st2 --version
diff --git a/test-requirements.txt b/test-requirements.txt
index 1ea72295cc..69909371f4 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -41,3 +41,5 @@ pytest-benchmark==3.2.3
 zstandard==0.15.1
 # ujson is used for micro benchmarks
 ujson==1.35
+# needed by integration tests for coordination
+redis==3.5.3

From 8f74e7a2958d32081f5020467e098a5303db7a56 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 19:26:34 +0100
Subject: [PATCH 07/29] Try listening only on ipv4.

---
 scripts/github/rabbitmq.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/github/rabbitmq.conf b/scripts/github/rabbitmq.conf
index 1c8032292b..76e281cfc7 100644
--- a/scripts/github/rabbitmq.conf
+++ b/scripts/github/rabbitmq.conf
@@ -1,5 +1,6 @@
 # bitnami/rabbitmq configuration file (gets merged with rabbitmq.conf)
 listeners.ssl.default            = 5671
+listeners.tcp.1 = 0.0.0.0:5672
 # /bitnami/conf is a directory mounted into the bitnami/rabbitmq container
 ssl_options.cacertfile           = /bitnami/conf/ssl_certs/ca/ca_certificate_bundle.pem
 ssl_options.certfile             = /bitnami/conf/ssl_certs/server/server_certificate.pem

From 35f2e2faae56aba7d222ff813b86cac618742093 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 19:26:52 +0100
Subject: [PATCH 08/29] For now, dont run micro benchmarks on nightly basis
 since it's very slow.

---
 Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index d5f0ce3319..8f210d4c39 100644
--- a/Makefile
+++ b/Makefile
@@ -280,7 +280,9 @@ check-python-packages-nightly:
 	done
 
 .PHONY: ci-checks-nightly
-ci-checks-nightly: check-python-packages-nightly micro-benchmarks
+# TODO: Ony run micro-benchmarks once a week since they are extremly slow on CI
+ci-checks-nightly: check-python-packages-nightly
+#ci-checks-nightly: check-python-packages-nightly micro-benchmarks
 
 .PHONY: checklogs
 checklogs:

From 1dfce18e28de7864c9a6981eda3fad6e1a03cd42 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 19:28:17 +0100
Subject: [PATCH 09/29] Use docker rm.

---
 .github/workflows/ci.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index f6ff0fe27f..2c09b57c62 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -320,7 +320,7 @@ jobs:
           ./scripts/ci/submit-codecov-coverage.sh
       - name: Stop Redis Service Container
         if: "${{ always() && env.TASK == 'ci-integration' }}"
-        run: docker stop redis || true
+        run: docker rm --force redis || true
   slack-notification:
     name: Slack notification for failed master builds
     if: always()

From cbbe3fba6c6a6f66141eedde33aa2b308c2e7b13 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 19:29:55 +0100
Subject: [PATCH 10/29] Remove workarounds we don't need anymore.

---
 Makefile | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/Makefile b/Makefile
index 8f210d4c39..32c772443e 100644
--- a/Makefile
+++ b/Makefile
@@ -649,16 +649,6 @@ requirements: virtualenv .requirements .sdist-requirements install-runners insta
 	$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)"  # workaround for pbr issue
 	$(VIRTUALENV_DIR)/bin/pip install --upgrade "pbr==5.4.3"  # workaround for pbr issue
 
-	# Fix for Travis CI race
-	$(VIRTUALENV_DIR)/bin/pip install "six==1.12.0"
-
-	# Fix for Travis CI caching issue
-	if [[ "$(TRAVIS_EVENT_TYPE)" != "" ]]; then\
-		$(VIRTUALENV_DIR)/bin/pip uninstall -y "pytz" || echo "not installed"; \
-		$(VIRTUALENV_DIR)/bin/pip uninstall -y "python-dateutil" || echo "not installed"; \
-		$(VIRTUALENV_DIR)/bin/pip uninstall -y "orquesta" || echo "not installed"; \
-	fi
-
 	# Install requirements
 	for req in $(REQUIREMENTS); do \
 		echo "Installing $$req..." ; \

From aa10c02358786c0f6076e5a07b2b1b94441781ab Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 19:35:26 +0100
Subject: [PATCH 11/29] Remove testing change.

---
 scripts/github/rabbitmq.conf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/github/rabbitmq.conf b/scripts/github/rabbitmq.conf
index 76e281cfc7..1c8032292b 100644
--- a/scripts/github/rabbitmq.conf
+++ b/scripts/github/rabbitmq.conf
@@ -1,6 +1,5 @@
 # bitnami/rabbitmq configuration file (gets merged with rabbitmq.conf)
 listeners.ssl.default            = 5671
-listeners.tcp.1 = 0.0.0.0:5672
 # /bitnami/conf is a directory mounted into the bitnami/rabbitmq container
 ssl_options.cacertfile           = /bitnami/conf/ssl_certs/ca/ca_certificate_bundle.pem
 ssl_options.certfile             = /bitnami/conf/ssl_certs/server/server_certificate.pem

From 2248e71809804eaf96f32e3137b1f1d20f5711f7 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 19:55:37 +0100
Subject: [PATCH 12/29] Add two more chunks and a comment.

---
 .github/workflows/ci.yaml | 36 ++++++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 2c09b57c62..2fa8b90186 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -24,6 +24,8 @@ on:
 # ones and only run commands which are needed for some steps for those steps and
 # not for all
 jobs:
+  # Special job which automatically cancels old runs for the same branch, prevents runs for the
+  # same file set which has already passed, etc.
   pre_job:
     name: Skip Duplicate Jobs Pre Job
     runs-on: ubuntu-latest
@@ -44,6 +46,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # NOTE: To speed the CI run, we split unit and integration tests into multiple jobs where
+        # each job runs subset of tests.
         include:
           - name: 'Lint Checks'
             task: 'ci-checks'
@@ -62,39 +66,49 @@ jobs:
             python-version: '3.6'
           - name: 'Unit Tests (chunk 1)'
             task: 'ci-unit'
-            nosetests_node_total: 3
+            nosetests_node_total: 4
             nosetests_node_index: 0
             python-version: '3.6'
           - name: 'Unit Tests (chunk 2)'
             task: 'ci-unit'
-            nosetests_node_total: 3
+            nosetests_node_total: 4
             nosetests_node_index: 1
             python-version: '3.6'
           - name: 'Unit Tests (chunk 3)'
             task: 'ci-unit'
-            nosetests_node_total: 3
+            nosetests_node_total: 4
             nosetests_node_index: 2
             python-version: '3.6'
+          - name: 'Unit Tests (chunk 4)'
+            task: 'ci-unit'
+            nosetests_node_total: 4
+            nosetests_node_index: 3
+            python-version: '3.6'
           - name: 'Integration Tests (chunk 1)'
             task: 'ci-integration'
-            nosetests_node_total: 4
+            nosetests_node_total: 5
             nosetests_node_index: 0
             python-version: '3.6'
           - name: 'Integration Tests (chunk 2)'
             task: 'ci-integration'
-            nosetests_node_total: 4
+            nosetests_node_total: 5
             nosetests_node_index: 1
             python-version: '3.6'
           - name: 'Integration Tests (chunk 3)'
             task: 'ci-integration'
-            nosetests_node_total: 4
+            nosetests_node_total: 5
             nosetests_node_index: 2
             python-version: '3.6'
           - name: 'Integration Tests (chunk 4)'
             task: 'ci-integration'
-            nosetests_node_total: 4
+            nosetests_node_total: 5
             nosetests_node_index: 3
             python-version: '3.6'
+          - name: 'Integration Tests (chunk 5)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 4
+            python-version: '3.6'
           # This job is slow so we only run in on a daily basis
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
@@ -165,7 +179,7 @@ jobs:
       # We need to explicitly specify terminal width otherwise some CLI tests fail on container
       # environments where small terminal size is used.
       COLUMNS: '120'
-      PYLINT_CONCURRENCY: '2'
+      PYLINT_CONCURRENCY: '3'
 
       # CI st2.conf (with ST2_CI_USER user instead of stanley)
       ST2_CONF: 'conf/st2.ci.conf'
@@ -216,7 +230,8 @@ jobs:
           echo "::set-output name=year::$(/bin/date -u "+%Y")"
           echo "::set-output name=month::$(/bin/date -u "+%m")"
           echo "::set-output name=week::$(/bin/date -u "+%U")"
-      - uses: actions/cache@v2
+      - name: Python Dependencies
+        uses: actions/cache@v2
         with:
           path: |
             ~/.cache/pip
@@ -228,7 +243,8 @@ jobs:
           key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
           restore-keys: |
             ${{ runner.os }}-python-${{ matrix.python }}-
-      - uses: actions/cache@v2
+      - name: APT Dependencies
+        uses: actions/cache@v2
         with:
           path: |
             /var/cache/apt/archives/*.deb

From 31d966bff0f6214ca3174629fdc0b623095fed4f Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 21:24:36 +0100
Subject: [PATCH 13/29] Move pylint check from ci-checks to ci-compile to
 spread the load more evenly across different jobs and speed up the build.

---
 .github/workflows/ci.yaml | 4 ++--
 Makefile                  | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 2fa8b90186..c2905187b8 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -49,12 +49,12 @@ jobs:
         # NOTE: To speed the CI run, we split unit and integration tests into multiple jobs where
         # each job runs subset of tests.
         include:
-          - name: 'Lint Checks'
+          - name: 'Lint Checks (black, flake8, etc.)'
             task: 'ci-checks'
             nosetests_node_total: 1
             nosetests_node_index: 0
             python-version: '3.6'
-          - name: 'Compile'
+          - name: 'Compile (pip deps, pylint)'
             task: 'ci-compile'
             nosetests_node_total: 1
             nosetests_node_index: 0
diff --git a/Makefile b/Makefile
index 32c772443e..42f2238821 100644
--- a/Makefile
+++ b/Makefile
@@ -1081,8 +1081,10 @@ debs:
 .PHONY: ci
 ci: ci-checks ci-unit ci-integration ci-packs-tests
 
+# NOTE: pylint is moved to ci-compile so we more evenly spread the load across
+# various different jobs to make the whole workflow complete faster
 .PHONY: ci-checks
-ci-checks: .generated-files-check .shellcheck .black-check .pre-commit-checks .pylint .flake8 check-requirements check-sdist-requirements .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check check-python-packages
+ci-checks: .generated-files-check .shellcheck .black-check .pre-commit-checks .flake8 check-requirements check-sdist-requirements .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check check-python-packages
 
 .PHONY: .rst-check
 .rst-check:
@@ -1143,4 +1145,4 @@ ci-orquesta: .ci-prepare-integration .orquesta-itests-coverage-html
 ci-packs-tests: .packs-tests
 
 .PHONY: ci-compile
-ci-compile: check-dependency-conflicts compilepy3
+ci-compile: check-dependency-conflicts compilepy3 .pylint

From 84cea732b21524511f10086b892a91813ccdcd53 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 21:28:39 +0100
Subject: [PATCH 14/29] Try simplifying and unifying the requirements.

---
 .github/workflows/ci.yaml | 2 +-
 Makefile                  | 7 +------
 fixed-requirements.txt    | 2 +-
 requirements.txt          | 2 +-
 st2tests/requirements.txt | 2 +-
 test-requirements.txt     | 1 +
 6 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index c2905187b8..aa0686970e 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -54,7 +54,7 @@ jobs:
             nosetests_node_total: 1
             nosetests_node_index: 0
             python-version: '3.6'
-          - name: 'Compile (pip deps, pylint)'
+          - name: 'Compile (pip deps, pylint, etc.)'
             task: 'ci-compile'
             nosetests_node_total: 1
             nosetests_node_index: 0
diff --git a/Makefile b/Makefile
index 42f2238821..506933bd39 100644
--- a/Makefile
+++ b/Makefile
@@ -659,12 +659,7 @@ requirements: virtualenv .requirements .sdist-requirements install-runners insta
 	# NOTE: We pass --no-deps to the script so we don't install all the
 	# package dependencies which are already installed as part of "requirements"
 	# make targets. This speeds up the build
-	(cd st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps)
-
-	# Note: We install prance here and not as part of any component
-	# requirements.txt because it has a conflict with our dependency (requires
-	# new version of requests) which we cant resolve at this moment
-	$(VIRTUALENV_DIR)/bin/pip install "prance==0.15.0"
+	(cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps)
 
 	# Install st2common to register metrics drivers
 	# NOTE: We pass --no-deps to the script so we don't install all the
diff --git a/fixed-requirements.txt b/fixed-requirements.txt
index fe0b881b1d..b9efb2dd9b 100644
--- a/fixed-requirements.txt
+++ b/fixed-requirements.txt
@@ -54,7 +54,7 @@ zake==0.2.2
 # test requirements below
 bcrypt==3.1.7
 jinja2==2.11.3
-mock==2.0.0
+mock==3.0.5
 more-itertools==5.0.0
 nose-timer==0.7.5
 nose-parallel==0.3.1
diff --git a/requirements.txt b/requirements.txt
index de82a2f670..1567d2a92d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -28,7 +28,7 @@ jsonpath-rw==1.4.0
 jsonschema==2.6.0
 kombu==4.6.6
 lockfile==0.12.2
-mock==2.0.0
+mock==3.0.5
 mongoengine==0.18.2
 more-itertools==5.0.0
 networkx==1.11
diff --git a/st2tests/requirements.txt b/st2tests/requirements.txt
index ab8092f93e..b0afe3e77f 100644
--- a/st2tests/requirements.txt
+++ b/st2tests/requirements.txt
@@ -6,7 +6,7 @@
 # in-requirements.txt for that component and then run 'make requirements' to
 # update the component requirements.txt
 RandomWords
-mock==2.0.0
+mock==3.0.5
 nose
 nose-parallel==0.3.1
 nose-timer==0.7.5
diff --git a/test-requirements.txt b/test-requirements.txt
index 69909371f4..f4f0116b7e 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -30,6 +30,7 @@ webtest==2.0.25
 rstcheck>=3.3.1,<3.4
 tox==3.14.1
 pyrabbit
+prance==0.15.0
 # pip-tools provides pip-compile: to check for version conflicts
 # pip-tools 5.3 needs pip<20.3
 # pip-tools 5.4 needs pip>=20.1

From 481dab7ecdc7f96c245808427cae291be99f6ca0 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 23:15:49 +0100
Subject: [PATCH 15/29] Don't skip duplicated builds for master branch.

---
 .github/workflows/ci.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index aa0686970e..193f29705e 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -40,7 +40,9 @@ jobs:
 
   ci:
     needs: pre_job
-    if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+    # NOTE: We always want to run job on master since we run some additional checks there (code
+    # coverage, etc)
+    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/master' }}
     name: '${{ matrix.name }} - python (${{ matrix.python-version }})'
     runs-on: ubuntu-latest
     strategy:

From 9e9bcc34500c2ad31b8a53ca101e49383b638a51 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 23:26:31 +0100
Subject: [PATCH 16/29] Test approach to cache apt deps to speed up the build.

---
 .github/workflows/ci.yaml | 156 +++++++++++++++++++++-----------------
 1 file changed, 86 insertions(+), 70 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 193f29705e..78a3cb179c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -56,61 +56,61 @@ jobs:
             nosetests_node_total: 1
             nosetests_node_index: 0
             python-version: '3.6'
-          - name: 'Compile (pip deps, pylint, etc.)'
-            task: 'ci-compile'
-            nosetests_node_total: 1
-            nosetests_node_index: 0
-            python-version: '3.6'
-          - name: 'Pack Tests'
-            task: 'ci-packs-tests'
-            nosetests_node_total: 1
-            nosetests_node_index: 0
-            python-version: '3.6'
-          - name: 'Unit Tests (chunk 1)'
-            task: 'ci-unit'
-            nosetests_node_total: 4
-            nosetests_node_index: 0
-            python-version: '3.6'
-          - name: 'Unit Tests (chunk 2)'
-            task: 'ci-unit'
-            nosetests_node_total: 4
-            nosetests_node_index: 1
-            python-version: '3.6'
-          - name: 'Unit Tests (chunk 3)'
-            task: 'ci-unit'
-            nosetests_node_total: 4
-            nosetests_node_index: 2
-            python-version: '3.6'
-          - name: 'Unit Tests (chunk 4)'
-            task: 'ci-unit'
-            nosetests_node_total: 4
-            nosetests_node_index: 3
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 1)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 0
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 2)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 1
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 3)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 2
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 4)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 3
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 5)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 4
-            python-version: '3.6'
+              # - name: 'Compile (pip deps, pylint, etc.)'
+              #   task: 'ci-compile'
+              #   nosetests_node_total: 1
+              #   nosetests_node_index: 0
+              #   python-version: '3.6'
+              # - name: 'Pack Tests'
+              #   task: 'ci-packs-tests'
+              #   nosetests_node_total: 1
+              #   nosetests_node_index: 0
+              #   python-version: '3.6'
+              # - name: 'Unit Tests (chunk 1)'
+              #   task: 'ci-unit'
+              #   nosetests_node_total: 4
+              #   nosetests_node_index: 0
+              #   python-version: '3.6'
+              # - name: 'Unit Tests (chunk 2)'
+              #   task: 'ci-unit'
+              #   nosetests_node_total: 4
+              #   nosetests_node_index: 1
+              #   python-version: '3.6'
+              # - name: 'Unit Tests (chunk 3)'
+              #   task: 'ci-unit'
+              #   nosetests_node_total: 4
+              #   nosetests_node_index: 2
+              #   python-version: '3.6'
+              # - name: 'Unit Tests (chunk 4)'
+              #   task: 'ci-unit'
+              #   nosetests_node_total: 4
+              #   nosetests_node_index: 3
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 1)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 0
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 2)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 1
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 3)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 2
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 4)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 3
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 5)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 4
+              #   python-version: '3.6'
           # This job is slow so we only run in on a daily basis
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
@@ -232,7 +232,8 @@ jobs:
           echo "::set-output name=year::$(/bin/date -u "+%Y")"
           echo "::set-output name=month::$(/bin/date -u "+%m")"
           echo "::set-output name=week::$(/bin/date -u "+%U")"
-      - name: Python Dependencies
+      - name: Cache Python Dependencies
+        id: cache-apt-deps
         uses: actions/cache@v2
         with:
           path: |
@@ -245,25 +246,40 @@ jobs:
           key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
           restore-keys: |
             ${{ runner.os }}-python-${{ matrix.python }}-
-      - name: APT Dependencies
+      - name: Cache APT Dependencies
         uses: actions/cache@v2
         with:
           path: |
-            /var/cache/apt/archives/*.deb
-            /var/cache/apt/archives/partial/*.deb
-            /var/cache/apt/*.bin
-          key: ${{ runner.os }}-apt-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
+            ~/apt_cache
+          key: ${{ runner.os }}-apt-v2-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
           restore-keys: |
-            ${{ runner.os }}-apt-${{ steps.date.outputs.year }}-
-            ${{ runner.os }}-apt-
-      - name: Install apt depedencies
+            ${{ runner.os }}-apt-v2-${{ steps.date.outputs.year }}-
+            ${{ runner.os }}-apt-v2-
+      - name: Install APT Depedencies
+        env:
+          CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
         run: |
-          # install dev dependencies for Python YAML and LDAP packages
-          # https://github.com/StackStorm/st2-auth-ldap
-          sudo apt-get -y update
-          sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
-          # shellcheck is already available in docker runner image we use
-          # sudo apt-get -y install shellcheck
+          if [[ "$CACHE_HIT" == 'true' ]]; then
+            echo "Restoring cached apt packages data from ~/apt_cache"
+            sudo cp --force --recursive ~/apt_cache/* /
+          else
+            sudo apt-get update && sudo apt-get install -y libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
+
+            mkdir -p ~/apt_cache
+
+            for dep in libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils; do
+                echo "Caching package files for package ${deb} into ~/apt_cache/"
+                dpkg -L $dep | while IFS= read -r f; do if test -f $f; then echo $f; fi; done | xargs cp --parents --target-directory ~/apt_cache/
+            done
+          fi
+          # - name: Install APT Depedencies
+          #   run: |
+          #     # install dev dependencies for Python YAML and LDAP packages
+          #     # https://github.com/StackStorm/st2-auth-ldap
+          #     sudo apt-get -y update
+          #     sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
+          #     # shellcheck is already available in docker runner image we use
+          #     # sudo apt-get -y install shellcheck
       - name: Install virtualenv
         run: |
           set -x

From c4a2538db39ea3b69fc09f323621a80a67793e21 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sat, 27 Mar 2021 23:49:23 +0100
Subject: [PATCH 17/29] Move functionality into a script file.

---
 .github/workflows/ci.yaml                     | 20 +++---------
 .../github/install-apt-packages-from-cache.sh | 31 +++++++++++++++++++
 2 files changed, 35 insertions(+), 16 deletions(-)
 create mode 100755 scripts/github/install-apt-packages-from-cache.sh

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 78a3cb179c..b660cd2950 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -233,7 +233,6 @@ jobs:
           echo "::set-output name=month::$(/bin/date -u "+%m")"
           echo "::set-output name=week::$(/bin/date -u "+%U")"
       - name: Cache Python Dependencies
-        id: cache-apt-deps
         uses: actions/cache@v2
         with:
           path: |
@@ -247,10 +246,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-python-${{ matrix.python }}-
       - name: Cache APT Dependencies
+        id: cache-apt-deps
         uses: actions/cache@v2
         with:
-          path: |
-            ~/apt_cache
+          path: "~/apt_cache"
+          # TODO: Also incorporate package names into cache key
           key: ${{ runner.os }}-apt-v2-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
           restore-keys: |
             ${{ runner.os }}-apt-v2-${{ steps.date.outputs.year }}-
@@ -259,19 +259,7 @@ jobs:
         env:
           CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
         run: |
-          if [[ "$CACHE_HIT" == 'true' ]]; then
-            echo "Restoring cached apt packages data from ~/apt_cache"
-            sudo cp --force --recursive ~/apt_cache/* /
-          else
-            sudo apt-get update && sudo apt-get install -y libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
-
-            mkdir -p ~/apt_cache
-
-            for dep in libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils; do
-                echo "Caching package files for package ${deb} into ~/apt_cache/"
-                dpkg -L $dep | while IFS= read -r f; do if test -f $f; then echo $f; fi; done | xargs cp --parents --target-directory ~/apt_cache/
-            done
-          fi
+          ./scripts/github/install-apt-packages-from-cache.sh
           # - name: Install APT Depedencies
           #   run: |
           #     # install dev dependencies for Python YAML and LDAP packages
diff --git a/scripts/github/install-apt-packages-from-cache.sh b/scripts/github/install-apt-packages-from-cache.sh
new file mode 100755
index 0000000000..470a9e8127
--- /dev/null
+++ b/scripts/github/install-apt-packages-from-cache.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Special script which supports installing apt-packages, caching installes files into a directory
+# and then on next run if cache is available, directly copying installed files to speed up the
+# build
+
+# Packages which will be installed and cached
+APT_PACKAGES="libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils"
+
+# Directory where installed package files will be copied - should match directory specified for
+# cache target in github actions workflow
+CACHE_DIRECTORY="${HOME}/apt_cache"
+
+if [[ "$CACHE_HIT" == 'true' ]]; then
+    echo "Restoring cached apt packages data from ${CACHE_DIRECTORY}"
+    sudo cp --force --recursive ~/apt_cache/* /
+else
+    echo "No cache found, installing packages using apt-get"
+    # shellcheck disable=SC2086
+    sudo apt-get update && sudo apt-get install -y ${APT_PACKAGES}
+
+    mkdir -p "${CACHE_DIRECTORY}"
+
+    for dep in ${APT_PACKAGES}; do
+        echo "Caching package files for package ${dep} into ${CACHE_DIRECTORY}"
+        dpkg -L "$dep" | while IFS= read -r f; do if test -f "$f"; then echo "$f"; fi; done | xargs cp --parents --target-directory "${CACHE_DIRECTORY}"
+    done
+
+    ls -la "${CACHE_DIRECTORY}"
+    du -hs "${CACHE_DIRECTORY}"
+fi

From ca7c7f2ec8b388cd9cdfb1756b7d3e8df2ea738e Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sun, 28 Mar 2021 00:15:03 +0100
Subject: [PATCH 18/29] Revert apt-cache change which doesn't seem to be
 working and it's hanging the whole github build.

---
 .github/workflows/ci.yaml | 138 +++++++++++++++++++-------------------
 1 file changed, 68 insertions(+), 70 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index b660cd2950..5c56cc0e2c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -56,61 +56,61 @@ jobs:
             nosetests_node_total: 1
             nosetests_node_index: 0
             python-version: '3.6'
-              # - name: 'Compile (pip deps, pylint, etc.)'
-              #   task: 'ci-compile'
-              #   nosetests_node_total: 1
-              #   nosetests_node_index: 0
-              #   python-version: '3.6'
-              # - name: 'Pack Tests'
-              #   task: 'ci-packs-tests'
-              #   nosetests_node_total: 1
-              #   nosetests_node_index: 0
-              #   python-version: '3.6'
-              # - name: 'Unit Tests (chunk 1)'
-              #   task: 'ci-unit'
-              #   nosetests_node_total: 4
-              #   nosetests_node_index: 0
-              #   python-version: '3.6'
-              # - name: 'Unit Tests (chunk 2)'
-              #   task: 'ci-unit'
-              #   nosetests_node_total: 4
-              #   nosetests_node_index: 1
-              #   python-version: '3.6'
-              # - name: 'Unit Tests (chunk 3)'
-              #   task: 'ci-unit'
-              #   nosetests_node_total: 4
-              #   nosetests_node_index: 2
-              #   python-version: '3.6'
-              # - name: 'Unit Tests (chunk 4)'
-              #   task: 'ci-unit'
-              #   nosetests_node_total: 4
-              #   nosetests_node_index: 3
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 1)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 0
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 2)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 1
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 3)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 2
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 4)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 3
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 5)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 4
-              #   python-version: '3.6'
+          - name: 'Compile (pip deps, pylint, etc.)'
+            task: 'ci-compile'
+            nosetests_node_total: 1
+            nosetests_node_index: 0
+            python-version: '3.6'
+          - name: 'Pack Tests'
+            task: 'ci-packs-tests'
+            nosetests_node_total: 1
+            nosetests_node_index: 0
+            python-version: '3.6'
+          - name: 'Unit Tests (chunk 1)'
+            task: 'ci-unit'
+            nosetests_node_total: 4
+            nosetests_node_index: 0
+            python-version: '3.6'
+          - name: 'Unit Tests (chunk 2)'
+            task: 'ci-unit'
+            nosetests_node_total: 4
+            nosetests_node_index: 1
+            python-version: '3.6'
+          - name: 'Unit Tests (chunk 3)'
+            task: 'ci-unit'
+            nosetests_node_total: 4
+            nosetests_node_index: 2
+            python-version: '3.6'
+          - name: 'Unit Tests (chunk 4)'
+            task: 'ci-unit'
+            nosetests_node_total: 4
+            nosetests_node_index: 3
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 1)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 0
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 2)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 1
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 3)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 2
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 4)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 3
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 5)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 4
+            python-version: '3.6'
           # This job is slow so we only run in on a daily basis
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
@@ -249,25 +249,23 @@ jobs:
         id: cache-apt-deps
         uses: actions/cache@v2
         with:
-          path: "~/apt_cache"
+          path: |
+            /var/cache/apt/archives/*.deb
+            /var/cache/apt/archives/partial/*.deb
+            /var/cache/apt/*.bin
           # TODO: Also incorporate package names into cache key
-          key: ${{ runner.os }}-apt-v2-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
+          key: ${{ runner.os }}-apt-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
           restore-keys: |
-            ${{ runner.os }}-apt-v2-${{ steps.date.outputs.year }}-
-            ${{ runner.os }}-apt-v2-
+            ${{ runner.os }}-apt-v3-${{ steps.date.outputs.year }}-
+            ${{ runner.os }}-apt-v3-
       - name: Install APT Depedencies
-        env:
-          CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
         run: |
-          ./scripts/github/install-apt-packages-from-cache.sh
-          # - name: Install APT Depedencies
-          #   run: |
-          #     # install dev dependencies for Python YAML and LDAP packages
-          #     # https://github.com/StackStorm/st2-auth-ldap
-          #     sudo apt-get -y update
-          #     sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
-          #     # shellcheck is already available in docker runner image we use
-          #     # sudo apt-get -y install shellcheck
+          # install dev dependencies for Python YAML and LDAP packages
+          # https://github.com/StackStorm/st2-auth-ldap
+          sudo apt-get -y update
+          sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
+          # shellcheck is already available in docker runner image we use
+          # sudo apt-get -y install shellcheck
       - name: Install virtualenv
         run: |
           set -x

From bffe4c55d88442058f917dbddb0d65bceadd9a67 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sun, 28 Mar 2021 00:15:28 +0100
Subject: [PATCH 19/29] Add new make target for running microbenchmarks as part
 of a weekly build.

---
 .github/workflows/ci.yaml | 2 +-
 Makefile                  | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 5c56cc0e2c..8ced8e612b 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -181,7 +181,7 @@ jobs:
       # We need to explicitly specify terminal width otherwise some CLI tests fail on container
       # environments where small terminal size is used.
       COLUMNS: '120'
-      PYLINT_CONCURRENCY: '3'
+      PYLINT_CONCURRENCY: '4'
 
       # CI st2.conf (with ST2_CI_USER user instead of stanley)
       ST2_CONF: 'conf/st2.ci.conf'
diff --git a/Makefile b/Makefile
index 506933bd39..3b0c3ba2b2 100644
--- a/Makefile
+++ b/Makefile
@@ -284,6 +284,10 @@ check-python-packages-nightly:
 ci-checks-nightly: check-python-packages-nightly
 #ci-checks-nightly: check-python-packages-nightly micro-benchmarks
 
+# CI checks which are very slow and only run on a weekly basic
+.PHONY: ci-checks-weekly
+ci-checks-weekly: micro-benchmarks
+
 .PHONY: checklogs
 checklogs:
 	@echo
@@ -646,8 +650,8 @@ requirements: virtualenv .requirements .sdist-requirements install-runners insta
 	#       only have to update it one place when we change the version
 	$(VIRTUALENV_DIR)/bin/pip install --upgrade $(shell grep "^virtualenv" fixed-requirements.txt)
 
-	$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)"  # workaround for pbr issue
-	$(VIRTUALENV_DIR)/bin/pip install --upgrade "pbr==5.4.3"  # workaround for pbr issue
+	#$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)"  # workaround for pbr issue
+	#$(VIRTUALENV_DIR)/bin/pip install --upgrade "pbr==5.4.3"  # workaround for pbr issue
 
 	# Install requirements
 	for req in $(REQUIREMENTS); do \

From ef4607f4e93bc36e5b7c1d5701fcbf5804a58b2b Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sun, 28 Mar 2021 00:26:14 +0100
Subject: [PATCH 20/29] Test another change.

---
 .github/workflows/ci.yaml                     | 126 +++++++++---------
 .../github/install-apt-packages-use-cache.sh  |  26 ++++
 2 files changed, 87 insertions(+), 65 deletions(-)
 create mode 100755 scripts/github/install-apt-packages-use-cache.sh

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 8ced8e612b..38a7d1c5d3 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -56,61 +56,61 @@ jobs:
             nosetests_node_total: 1
             nosetests_node_index: 0
             python-version: '3.6'
-          - name: 'Compile (pip deps, pylint, etc.)'
-            task: 'ci-compile'
-            nosetests_node_total: 1
-            nosetests_node_index: 0
-            python-version: '3.6'
-          - name: 'Pack Tests'
-            task: 'ci-packs-tests'
-            nosetests_node_total: 1
-            nosetests_node_index: 0
-            python-version: '3.6'
-          - name: 'Unit Tests (chunk 1)'
-            task: 'ci-unit'
-            nosetests_node_total: 4
-            nosetests_node_index: 0
-            python-version: '3.6'
-          - name: 'Unit Tests (chunk 2)'
-            task: 'ci-unit'
-            nosetests_node_total: 4
-            nosetests_node_index: 1
-            python-version: '3.6'
-          - name: 'Unit Tests (chunk 3)'
-            task: 'ci-unit'
-            nosetests_node_total: 4
-            nosetests_node_index: 2
-            python-version: '3.6'
-          - name: 'Unit Tests (chunk 4)'
-            task: 'ci-unit'
-            nosetests_node_total: 4
-            nosetests_node_index: 3
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 1)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 0
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 2)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 1
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 3)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 2
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 4)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 3
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 5)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 4
-            python-version: '3.6'
+              # - name: 'Compile (pip deps, pylint, etc.)'
+              #   task: 'ci-compile'
+              #   nosetests_node_total: 1
+              #   nosetests_node_index: 0
+              #   python-version: '3.6'
+              # - name: 'Pack Tests'
+              #   task: 'ci-packs-tests'
+              #   nosetests_node_total: 1
+              #   nosetests_node_index: 0
+              #   python-version: '3.6'
+              # - name: 'Unit Tests (chunk 1)'
+              #   task: 'ci-unit'
+              #   nosetests_node_total: 4
+              #   nosetests_node_index: 0
+              #   python-version: '3.6'
+              # - name: 'Unit Tests (chunk 2)'
+              #   task: 'ci-unit'
+              #   nosetests_node_total: 4
+              #   nosetests_node_index: 1
+              #   python-version: '3.6'
+              # - name: 'Unit Tests (chunk 3)'
+              #   task: 'ci-unit'
+              #   nosetests_node_total: 4
+              #   nosetests_node_index: 2
+              #   python-version: '3.6'
+              # - name: 'Unit Tests (chunk 4)'
+              #   task: 'ci-unit'
+              #   nosetests_node_total: 4
+              #   nosetests_node_index: 3
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 1)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 0
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 2)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 1
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 3)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 2
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 4)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 3
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 5)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 4
+              #   python-version: '3.6'
           # This job is slow so we only run in on a daily basis
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
@@ -250,22 +250,17 @@ jobs:
         uses: actions/cache@v2
         with:
           path: |
-            /var/cache/apt/archives/*.deb
-            /var/cache/apt/archives/partial/*.deb
-            /var/cache/apt/*.bin
+            ~/apt_cache
           # TODO: Also incorporate package names into cache key
-          key: ${{ runner.os }}-apt-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
+          key: ${{ runner.os }}-apt-v4-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
           restore-keys: |
-            ${{ runner.os }}-apt-v3-${{ steps.date.outputs.year }}-
-            ${{ runner.os }}-apt-v3-
+            ${{ runner.os }}-apt-v4-${{ steps.date.outputs.year }}-
+            ${{ runner.os }}-apt-v4-
       - name: Install APT Depedencies
         run: |
           # install dev dependencies for Python YAML and LDAP packages
           # https://github.com/StackStorm/st2-auth-ldap
-          sudo apt-get -y update
-          sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
-          # shellcheck is already available in docker runner image we use
-          # sudo apt-get -y install shellcheck
+          ./scripts/github/install-apt-packages-use-cache.sh
       - name: Install virtualenv
         run: |
           set -x
@@ -327,6 +322,7 @@ jobs:
       - name: make
         # use: script -e -c to print colors
         run: |
+          exit 0
           script -e -c "make ${TASK}"
       - name: Nightly
         # Run any additional nightly checks only as part of a nightly (cron) build
diff --git a/scripts/github/install-apt-packages-use-cache.sh b/scripts/github/install-apt-packages-use-cache.sh
new file mode 100755
index 0000000000..0db1ca409d
--- /dev/null
+++ b/scripts/github/install-apt-packages-use-cache.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# Special script which supports installing apt-packages, caching installed files into a directory
+# and then on next run if cache is available, re-using that cache
+
+# Packages which will be installed and cached
+# NOTE: shellcheck is already available in docker runner image we use
+APT_PACKAGES="libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils"
+
+# Directory where installed package files will be copied - should match directory specified for
+# cache target in github actions workflow
+CACHE_DIRECTORY="${HOME}/apt_cache"
+
+export APT_DIR="${CACHE_DIRECTORY}"
+export APT_STATE_LISTS="${APT_DIR}/lists"
+export APT_CACHE_ARCHIVES="${APT_DIR}/archives"
+
+# shellcheck disable=SC2059
+printf "dir::state::lists    ${APT_STATE_LISTS};\ndir::cache::archives    ${APT_CACHE_ARCHIVES};\n" | sudo tee /etc/apt/apt.conf
+
+mkdir -p "${APT_STATE_LISTS}/partial"
+mkdir -p "${APT_CACHE_ARCHIVES}/partial"
+
+sudo apt-get -y update
+# shellcheck disable=SC2086
+sudo apt-get -f -y install ${APT_PACKAGES}

From 71a2265407a497cdd07757f7c4f0c2a8660698b1 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sun, 28 Mar 2021 00:49:17 +0100
Subject: [PATCH 21/29] Remove testing changes.

---
 .github/workflows/ci.yaml | 111 +++++++++++++++++++-------------------
 1 file changed, 55 insertions(+), 56 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 38a7d1c5d3..2544401acd 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -56,61 +56,61 @@ jobs:
             nosetests_node_total: 1
             nosetests_node_index: 0
             python-version: '3.6'
-              # - name: 'Compile (pip deps, pylint, etc.)'
-              #   task: 'ci-compile'
-              #   nosetests_node_total: 1
-              #   nosetests_node_index: 0
-              #   python-version: '3.6'
-              # - name: 'Pack Tests'
-              #   task: 'ci-packs-tests'
-              #   nosetests_node_total: 1
-              #   nosetests_node_index: 0
-              #   python-version: '3.6'
-              # - name: 'Unit Tests (chunk 1)'
-              #   task: 'ci-unit'
-              #   nosetests_node_total: 4
-              #   nosetests_node_index: 0
-              #   python-version: '3.6'
-              # - name: 'Unit Tests (chunk 2)'
-              #   task: 'ci-unit'
-              #   nosetests_node_total: 4
-              #   nosetests_node_index: 1
-              #   python-version: '3.6'
-              # - name: 'Unit Tests (chunk 3)'
-              #   task: 'ci-unit'
-              #   nosetests_node_total: 4
-              #   nosetests_node_index: 2
-              #   python-version: '3.6'
-              # - name: 'Unit Tests (chunk 4)'
-              #   task: 'ci-unit'
-              #   nosetests_node_total: 4
-              #   nosetests_node_index: 3
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 1)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 0
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 2)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 1
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 3)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 2
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 4)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 3
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 5)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 4
-              #   python-version: '3.6'
+          - name: 'Compile (pip deps, pylint, etc.)'
+            task: 'ci-compile'
+            nosetests_node_total: 1
+            nosetests_node_index: 0
+            python-version: '3.6'
+          - name: 'Pack Tests'
+            task: 'ci-packs-tests'
+            nosetests_node_total: 1
+            nosetests_node_index: 0
+            python-version: '3.6'
+          - name: 'Unit Tests (chunk 1)'
+            task: 'ci-unit'
+            nosetests_node_total: 4
+            nosetests_node_index: 0
+            python-version: '3.6'
+          - name: 'Unit Tests (chunk 2)'
+            task: 'ci-unit'
+            nosetests_node_total: 4
+            nosetests_node_index: 1
+            python-version: '3.6'
+          - name: 'Unit Tests (chunk 3)'
+            task: 'ci-unit'
+            nosetests_node_total: 4
+            nosetests_node_index: 2
+            python-version: '3.6'
+          - name: 'Unit Tests (chunk 4)'
+            task: 'ci-unit'
+            nosetests_node_total: 4
+            nosetests_node_index: 3
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 1)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 0
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 2)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 1
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 3)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 2
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 4)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 3
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 5)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 4
+            python-version: '3.6'
           # This job is slow so we only run in on a daily basis
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
@@ -322,7 +322,6 @@ jobs:
       - name: make
         # use: script -e -c to print colors
         run: |
-          exit 0
           script -e -c "make ${TASK}"
       - name: Nightly
         # Run any additional nightly checks only as part of a nightly (cron) build

From d755ba10fdfbab063756a57f9a455dfed4279540 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sun, 28 Mar 2021 13:39:08 +0200
Subject: [PATCH 22/29] Test a change.

---
 .github/workflows/ci.yaml                     | 111 +++++++++---------
 .../github/install-apt-packages-use-cache.sh  |   9 ++
 2 files changed, 65 insertions(+), 55 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 2544401acd..38a7d1c5d3 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -56,61 +56,61 @@ jobs:
             nosetests_node_total: 1
             nosetests_node_index: 0
             python-version: '3.6'
-          - name: 'Compile (pip deps, pylint, etc.)'
-            task: 'ci-compile'
-            nosetests_node_total: 1
-            nosetests_node_index: 0
-            python-version: '3.6'
-          - name: 'Pack Tests'
-            task: 'ci-packs-tests'
-            nosetests_node_total: 1
-            nosetests_node_index: 0
-            python-version: '3.6'
-          - name: 'Unit Tests (chunk 1)'
-            task: 'ci-unit'
-            nosetests_node_total: 4
-            nosetests_node_index: 0
-            python-version: '3.6'
-          - name: 'Unit Tests (chunk 2)'
-            task: 'ci-unit'
-            nosetests_node_total: 4
-            nosetests_node_index: 1
-            python-version: '3.6'
-          - name: 'Unit Tests (chunk 3)'
-            task: 'ci-unit'
-            nosetests_node_total: 4
-            nosetests_node_index: 2
-            python-version: '3.6'
-          - name: 'Unit Tests (chunk 4)'
-            task: 'ci-unit'
-            nosetests_node_total: 4
-            nosetests_node_index: 3
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 1)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 0
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 2)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 1
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 3)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 2
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 4)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 3
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 5)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 4
-            python-version: '3.6'
+              # - name: 'Compile (pip deps, pylint, etc.)'
+              #   task: 'ci-compile'
+              #   nosetests_node_total: 1
+              #   nosetests_node_index: 0
+              #   python-version: '3.6'
+              # - name: 'Pack Tests'
+              #   task: 'ci-packs-tests'
+              #   nosetests_node_total: 1
+              #   nosetests_node_index: 0
+              #   python-version: '3.6'
+              # - name: 'Unit Tests (chunk 1)'
+              #   task: 'ci-unit'
+              #   nosetests_node_total: 4
+              #   nosetests_node_index: 0
+              #   python-version: '3.6'
+              # - name: 'Unit Tests (chunk 2)'
+              #   task: 'ci-unit'
+              #   nosetests_node_total: 4
+              #   nosetests_node_index: 1
+              #   python-version: '3.6'
+              # - name: 'Unit Tests (chunk 3)'
+              #   task: 'ci-unit'
+              #   nosetests_node_total: 4
+              #   nosetests_node_index: 2
+              #   python-version: '3.6'
+              # - name: 'Unit Tests (chunk 4)'
+              #   task: 'ci-unit'
+              #   nosetests_node_total: 4
+              #   nosetests_node_index: 3
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 1)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 0
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 2)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 1
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 3)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 2
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 4)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 3
+              #   python-version: '3.6'
+              # - name: 'Integration Tests (chunk 5)'
+              #   task: 'ci-integration'
+              #   nosetests_node_total: 5
+              #   nosetests_node_index: 4
+              #   python-version: '3.6'
           # This job is slow so we only run in on a daily basis
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
@@ -322,6 +322,7 @@ jobs:
       - name: make
         # use: script -e -c to print colors
         run: |
+          exit 0
           script -e -c "make ${TASK}"
       - name: Nightly
         # Run any additional nightly checks only as part of a nightly (cron) build
diff --git a/scripts/github/install-apt-packages-use-cache.sh b/scripts/github/install-apt-packages-use-cache.sh
index 0db1ca409d..6b68108e2c 100755
--- a/scripts/github/install-apt-packages-use-cache.sh
+++ b/scripts/github/install-apt-packages-use-cache.sh
@@ -21,6 +21,15 @@ printf "dir::state::lists    ${APT_STATE_LISTS};\ndir::cache::archives    ${APT_
 mkdir -p "${APT_STATE_LISTS}/partial"
 mkdir -p "${APT_CACHE_ARCHIVES}/partial"
 
+ls -la "${APT_STATE_LISTS}"
+ls -la "${APT_CACHE_ARCHIVES}"
+
 sudo apt-get -y update
 # shellcheck disable=SC2086
 sudo apt-get -f -y install ${APT_PACKAGES}
+
+ls -la "${APT_STATE_LISTS}"
+ls -la "${APT_CACHE_ARCHIVES}"
+
+# Workaround for caching issue (ensure runer can read the downloaded packages)
+sudo chown -R runner:runner "${CACHE_DIRECTORY}"

From f0e0e6aacea65eafb59ab4e12fdcfd196983fdda Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sun, 28 Mar 2021 13:52:55 +0200
Subject: [PATCH 23/29] Change cache key.

---
 .github/workflows/ci.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 38a7d1c5d3..f23f0e2631 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -252,10 +252,10 @@ jobs:
           path: |
             ~/apt_cache
           # TODO: Also incorporate package names into cache key
-          key: ${{ runner.os }}-apt-v4-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
+          key: ${{ runner.os }}-apt-v5-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
           restore-keys: |
-            ${{ runner.os }}-apt-v4-${{ steps.date.outputs.year }}-
-            ${{ runner.os }}-apt-v4-
+            ${{ runner.os }}-apt-v5-${{ steps.date.outputs.year }}-
+            ${{ runner.os }}-apt-v5-
       - name: Install APT Depedencies
         run: |
           # install dev dependencies for Python YAML and LDAP packages

From 581a61e592c0f85ad146f87b07484ca7d0af7b65 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sun, 28 Mar 2021 13:59:46 +0200
Subject: [PATCH 24/29] Dont run apt-get update on populated cache.

---
 scripts/github/install-apt-packages-use-cache.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scripts/github/install-apt-packages-use-cache.sh b/scripts/github/install-apt-packages-use-cache.sh
index 6b68108e2c..e2963de3a1 100755
--- a/scripts/github/install-apt-packages-use-cache.sh
+++ b/scripts/github/install-apt-packages-use-cache.sh
@@ -21,10 +21,12 @@ printf "dir::state::lists    ${APT_STATE_LISTS};\ndir::cache::archives    ${APT_
 mkdir -p "${APT_STATE_LISTS}/partial"
 mkdir -p "${APT_CACHE_ARCHIVES}/partial"
 
-ls -la "${APT_STATE_LISTS}"
-ls -la "${APT_CACHE_ARCHIVES}"
+# NOTE apt-get update is only needed is there is no cache. If there is an existing cache, we don't
+# run it to speed things up
+if [[ "$CACHE_HIT" == 'false' ]]; then
+    sudo apt-get -y update
+fi
 
-sudo apt-get -y update
 # shellcheck disable=SC2086
 sudo apt-get -f -y install ${APT_PACKAGES}
 

From c97eb142bbbcfcd423574b4cd0f378ea28d89f97 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sun, 28 Mar 2021 14:05:44 +0200
Subject: [PATCH 25/29] Remove testing changes.

---
 .github/workflows/ci.yaml | 114 +++++++++++++++++++-------------------
 1 file changed, 57 insertions(+), 57 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index f23f0e2631..aba2819cbd 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -56,61 +56,61 @@ jobs:
             nosetests_node_total: 1
             nosetests_node_index: 0
             python-version: '3.6'
-              # - name: 'Compile (pip deps, pylint, etc.)'
-              #   task: 'ci-compile'
-              #   nosetests_node_total: 1
-              #   nosetests_node_index: 0
-              #   python-version: '3.6'
-              # - name: 'Pack Tests'
-              #   task: 'ci-packs-tests'
-              #   nosetests_node_total: 1
-              #   nosetests_node_index: 0
-              #   python-version: '3.6'
-              # - name: 'Unit Tests (chunk 1)'
-              #   task: 'ci-unit'
-              #   nosetests_node_total: 4
-              #   nosetests_node_index: 0
-              #   python-version: '3.6'
-              # - name: 'Unit Tests (chunk 2)'
-              #   task: 'ci-unit'
-              #   nosetests_node_total: 4
-              #   nosetests_node_index: 1
-              #   python-version: '3.6'
-              # - name: 'Unit Tests (chunk 3)'
-              #   task: 'ci-unit'
-              #   nosetests_node_total: 4
-              #   nosetests_node_index: 2
-              #   python-version: '3.6'
-              # - name: 'Unit Tests (chunk 4)'
-              #   task: 'ci-unit'
-              #   nosetests_node_total: 4
-              #   nosetests_node_index: 3
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 1)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 0
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 2)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 1
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 3)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 2
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 4)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 3
-              #   python-version: '3.6'
-              # - name: 'Integration Tests (chunk 5)'
-              #   task: 'ci-integration'
-              #   nosetests_node_total: 5
-              #   nosetests_node_index: 4
-              #   python-version: '3.6'
+          - name: 'Compile (pip deps, pylint, etc.)'
+            task: 'ci-compile'
+            nosetests_node_total: 1
+            nosetests_node_index: 0
+            python-version: '3.6'
+          - name: 'Pack Tests'
+            task: 'ci-packs-tests'
+            nosetests_node_total: 1
+            nosetests_node_index: 0
+            python-version: '3.6'
+          - name: 'Unit Tests (chunk 1)'
+            task: 'ci-unit'
+            nosetests_node_total: 4
+            nosetests_node_index: 0
+            python-version: '3.6'
+          - name: 'Unit Tests (chunk 2)'
+            task: 'ci-unit'
+            nosetests_node_total: 4
+            nosetests_node_index: 1
+            python-version: '3.6'
+          - name: 'Unit Tests (chunk 3)'
+            task: 'ci-unit'
+            nosetests_node_total: 4
+            nosetests_node_index: 2
+            python-version: '3.6'
+          - name: 'Unit Tests (chunk 4)'
+            task: 'ci-unit'
+            nosetests_node_total: 4
+            nosetests_node_index: 3
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 1)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 0
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 2)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 1
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 3)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 2
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 4)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 3
+            python-version: '3.6'
+          - name: 'Integration Tests (chunk 5)'
+            task: 'ci-integration'
+            nosetests_node_total: 5
+            nosetests_node_index: 4
+            python-version: '3.6'
           # This job is slow so we only run in on a daily basis
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
@@ -155,8 +155,6 @@ jobs:
           #
 
       # Used for the coordination backend for integration tests
-      # TODO: Only start this for integration tests via job step
-      # https://github.community/t/conditional-services-in-a-job/135301/3
       # NOTE: To speed things up, we only start redis for integration tests
       # where it's needed
       # redis:
@@ -257,6 +255,8 @@ jobs:
             ${{ runner.os }}-apt-v5-${{ steps.date.outputs.year }}-
             ${{ runner.os }}-apt-v5-
       - name: Install APT Depedencies
+        env:
+          CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
         run: |
           # install dev dependencies for Python YAML and LDAP packages
           # https://github.com/StackStorm/st2-auth-ldap

From 32798c1ca4209a4552c7593d8773a459faa6db53 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sun, 28 Mar 2021 14:08:12 +0200
Subject: [PATCH 26/29] Remove file we don't need anymore.

---
 .../github/install-apt-packages-from-cache.sh | 31 -------------------
 1 file changed, 31 deletions(-)
 delete mode 100755 scripts/github/install-apt-packages-from-cache.sh

diff --git a/scripts/github/install-apt-packages-from-cache.sh b/scripts/github/install-apt-packages-from-cache.sh
deleted file mode 100755
index 470a9e8127..0000000000
--- a/scripts/github/install-apt-packages-from-cache.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-
-# Special script which supports installing apt-packages, caching installes files into a directory
-# and then on next run if cache is available, directly copying installed files to speed up the
-# build
-
-# Packages which will be installed and cached
-APT_PACKAGES="libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils"
-
-# Directory where installed package files will be copied - should match directory specified for
-# cache target in github actions workflow
-CACHE_DIRECTORY="${HOME}/apt_cache"
-
-if [[ "$CACHE_HIT" == 'true' ]]; then
-    echo "Restoring cached apt packages data from ${CACHE_DIRECTORY}"
-    sudo cp --force --recursive ~/apt_cache/* /
-else
-    echo "No cache found, installing packages using apt-get"
-    # shellcheck disable=SC2086
-    sudo apt-get update && sudo apt-get install -y ${APT_PACKAGES}
-
-    mkdir -p "${CACHE_DIRECTORY}"
-
-    for dep in ${APT_PACKAGES}; do
-        echo "Caching package files for package ${dep} into ${CACHE_DIRECTORY}"
-        dpkg -L "$dep" | while IFS= read -r f; do if test -f "$f"; then echo "$f"; fi; done | xargs cp --parents --target-directory "${CACHE_DIRECTORY}"
-    done
-
-    ls -la "${CACHE_DIRECTORY}"
-    du -hs "${CACHE_DIRECTORY}"
-fi

From c6e0569ff19deb2c07995c53820051d6149ecbb1 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sun, 28 Mar 2021 14:09:00 +0200
Subject: [PATCH 27/29] Remove testing change.

---
 .github/workflows/ci.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index aba2819cbd..7e78fcc5d6 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -322,7 +322,6 @@ jobs:
       - name: make
         # use: script -e -c to print colors
         run: |
-          exit 0
           script -e -c "make ${TASK}"
       - name: Nightly
         # Run any additional nightly checks only as part of a nightly (cron) build

From 5bf8f735f89d2da0e913dab0818052a1526564f0 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Sun, 28 Mar 2021 17:00:14 +0200
Subject: [PATCH 28/29] Remove change which is not needed anymore.

---
 Makefile | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 3b0c3ba2b2..dccde6fe66 100644
--- a/Makefile
+++ b/Makefile
@@ -644,14 +644,14 @@ requirements: virtualenv .requirements .sdist-requirements install-runners insta
 	@echo "==================== requirements ===================="
 	@echo
 	# Show pip installed packages before we start
+	echo ""
 	$(VIRTUALENV_DIR)/bin/pip list
+	echo ""
 
 	# Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
 	#       only have to update it one place when we change the version
 	$(VIRTUALENV_DIR)/bin/pip install --upgrade $(shell grep "^virtualenv" fixed-requirements.txt)
-
-	#$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)"  # workaround for pbr issue
-	#$(VIRTUALENV_DIR)/bin/pip install --upgrade "pbr==5.4.3"  # workaround for pbr issue
+	$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)"  # workaround for pbr issue
 
 	# Install requirements
 	for req in $(REQUIREMENTS); do \
@@ -681,7 +681,9 @@ requirements: virtualenv .requirements .sdist-requirements install-runners insta
 	git submodule update --init --recursive --remote
 
 	# Show currently install requirements
+	echo ""
 	$(VIRTUALENV_DIR)/bin/pip list
+	echo ""
 
 .PHONY: check-dependency-conflicts
 check-dependency-conflicts:

From 23094ff513bd5246aa15f956ad4e867f7f60d561 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Wed, 31 Mar 2021 21:28:53 +0200
Subject: [PATCH 29/29] Decrease number of jobs which run in parallel.

---
 .github/workflows/ci.yaml | 27 ++++++---------------------
 1 file changed, 6 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 7e78fcc5d6..00589aed63 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -68,49 +68,34 @@ jobs:
             python-version: '3.6'
           - name: 'Unit Tests (chunk 1)'
             task: 'ci-unit'
-            nosetests_node_total: 4
+            nosetests_node_total: 3
             nosetests_node_index: 0
             python-version: '3.6'
           - name: 'Unit Tests (chunk 2)'
             task: 'ci-unit'
-            nosetests_node_total: 4
+            nosetests_node_total: 3
             nosetests_node_index: 1
             python-version: '3.6'
           - name: 'Unit Tests (chunk 3)'
             task: 'ci-unit'
-            nosetests_node_total: 4
+            nosetests_node_total: 3
             nosetests_node_index: 2
             python-version: '3.6'
-          - name: 'Unit Tests (chunk 4)'
-            task: 'ci-unit'
-            nosetests_node_total: 4
-            nosetests_node_index: 3
-            python-version: '3.6'
           - name: 'Integration Tests (chunk 1)'
             task: 'ci-integration'
-            nosetests_node_total: 5
+            nosetests_node_total: 3
             nosetests_node_index: 0
             python-version: '3.6'
           - name: 'Integration Tests (chunk 2)'
             task: 'ci-integration'
-            nosetests_node_total: 5
+            nosetests_node_total: 3
             nosetests_node_index: 1
             python-version: '3.6'
           - name: 'Integration Tests (chunk 3)'
             task: 'ci-integration'
-            nosetests_node_total: 5
+            nosetests_node_total: 3
             nosetests_node_index: 2
             python-version: '3.6'
-          - name: 'Integration Tests (chunk 4)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 3
-            python-version: '3.6'
-          - name: 'Integration Tests (chunk 5)'
-            task: 'ci-integration'
-            nosetests_node_total: 5
-            nosetests_node_index: 4
-            python-version: '3.6'
           # This job is slow so we only run in on a daily basis
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'