Python3.8 on Ubuntu 20.04 (requiring MongoDB 4.4)

.github/workflows/ci.yaml

@@ -55,6 +55,12 @@ jobs:
           - name: 'Compile (pip deps, pylint, etc.)'
             task: 'ci-compile'
             python-version: '3.6'
+          - name: 'Lint Checks (black, flake8, etc.)'
+            task: 'ci-checks'
+            python-version: '3.8'
+          - name: 'Compile (pip deps, pylint, etc.)'
+            task: 'ci-compile'
+            python-version: '3.8'
 
     env:
       TASK: '${{ matrix.task }}'
@@ -80,9 +86,9 @@ jobs:
           # TODO: maybe make the virtualenv a partial cache to exclude st2*?
           # !virtualenv/lib/python*/site-packages/st2*
           # !virtualenv/bin/st2*
-          key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
+          key: ${{ runner.os }}-v2-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
           restore-keys: |
-            ${{ runner.os }}-python-${{ matrix.python }}-
+            ${{ runner.os }}-v2-python-${{ matrix.python }}-
       - name: Cache APT Dependencies
         id: cache-apt-deps
         uses: actions/cache@v2
@@ -134,19 +140,24 @@ jobs:
         include:
           - name: 'Unit Tests (chunk 1)'
             task: 'ci-unit'
-            nosetests_node_total: 3
+            nosetests_node_total: 2
             nosetests_node_index: 0
             python-version: '3.6'
           - name: 'Unit Tests (chunk 2)'
             task: 'ci-unit'
-            nosetests_node_total: 3
+            nosetests_node_total: 2
             nosetests_node_index: 1
             python-version: '3.6'
-          - name: 'Unit Tests (chunk 3)'
+          - name: 'Unit Tests (chunk 1)'
             task: 'ci-unit'
-            nosetests_node_total: 3
-            nosetests_node_index: 2
-            python-version: '3.6'
+            nosetests_node_total: 2
+            nosetests_node_index: 0
+            python-version: '3.8'
+          - name: 'Unit Tests (chunk 2)'
+            task: 'ci-unit'
+            nosetests_node_total: 2
+            nosetests_node_index: 1
+            python-version: '3.8'
           # This job is slow so we only run in on a daily basis
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
@@ -155,7 +166,7 @@ jobs:
           #  nosetests_node_ index: 0
     services:
       mongo:
-        image: mongo:4.0
+        image: mongo:4.4
         ports:
           - 27017:27017
 
@@ -309,18 +320,24 @@ jobs:
             nosetests_node_total: 2
             nosetests_node_index: 1
             python-version: '3.6'
-          # There are a bunch of race conditions in Orquesta tests which causes
-          # the step to either fail or timeout quite often. We run those tests
-          # separately so we can re-run just that set of tests on failure /
-          # timeout instead of needing to re runn the whole build.
-          #- name: 'Integration Tests (Orquesta)'
-          #  task: 'ci-orquesta'
-          #  nosetests_node_total: 1
-          #  nosetests_node_index: 0
-          #  python-version: '3.6'
+          - name: 'Pack Tests'
+            task: 'ci-packs-tests'
+            nosetests_node_total: 1
+            nosetests_node_index: 0
+            python-version: '3.8'
+          - name: 'Integration Tests (chunk 1)'
+            task: 'ci-integration'
+            nosetests_node_total: 2
+            nosetests_node_index: 0
+            python-version: '3.8'
+          - name: 'Integration Tests (chunk 2)'
+            task: 'ci-integration'
+            nosetests_node_total: 2
+            nosetests_node_index: 1
+            python-version: '3.8'
     services:
       mongo:
-        image: mongo:4.0
+        image: mongo:4.4
         ports:
           - 27017:27017
 
@@ -466,13 +483,14 @@ jobs:
           ./scripts/github/configure-rabbitmq.sh
       - name: Print versions
         run: |
+
           ./scripts/ci/print-versions.sh
       - name: make
         if: "${{ env.TASK == 'ci-integration' }}"
         #timeout-minutes: 7
         # TODO: Use dynamic timeout value based on the branch - for master we
         # need to use timeout x2 due to coverage overhead
-        timeout-minutes: 14
+        timeout-minutes: 14  # may die if rabbitmq fails to start
         # use: script -e -c to print colors
         run: |
           script -e -c "make ${TASK}" && exit 0

.github/workflows/orquesta-integration-tests.yaml

@@ -55,9 +55,14 @@ jobs:
             nosetests_node_total: 1
             nosetests_node_index: 0
             python-version: '3.6'
+          - name: 'Integration Tests (Orquesta)'
+            task: 'ci-orquesta'
+            nosetests_node_total: 1
+            nosetests_node_index: 0
+            python-version: '3.8'
     services:
       mongo:
-        image: mongo:4.0
+        image: mongo:4.4
         ports:
           - 27017:27017
 

.pre-commit-config.yaml

@@ -2,7 +2,9 @@
 # files.
 # This hook relies on development virtual environment being present in virtualenv/.
 default_language_version:
-  python: python3.6
+  # We don't specify a specific version so it uses either Python 3.6 / Python
+  # 3.8 depending on what is available locally and on the runner
+  python: python
 
 exclude: '(build|dist)'
 repos:

CHANGELOG.rst

@@ -49,6 +49,22 @@ Added
 
   Contributed by @cognifloyd.
 
+* Add new ``database.compressors`` and ``database.zlib_compression_level`` config option which
+  specifies compression algorithms client supports for network / transport level compression
+  when talking to MongoDB.
+
+  Actual compression algorithm used will be then decided by the server and depends on the
+  algorithms which are supported by the server + client.
+
+  Possible / valid values include: zstd, zlib. Keep in mind that zstandard (zstd) is only supported
+  by MongoDB >= 4.2.
+
+  Our official Debian and RPM packages bundle ``zstandard`` dependency by default which means
+  setting this value to ``zstd`` should work out of the box as long as the server runs
+  MongoDB >= 4.2. #5177
+
+  Contributed by @Kami.
+
 Changed
 ~~~~~~~
 
@@ -172,6 +188,10 @@ Changed
 
   Contributed by @Kami.
 
+* Update code and dependencies so it supports Python 3.8 and Mongo DB 4.4 #5177
+
+  Contributed by @nzloshm @winem @Kami.
+
 * StackStorm Web UI (``st2web``) has been updated to not render and display execution results
   larger than 200 KB directly in the history panel in the right side bar by default anymore.
   Instead a link to view or download the raw result is displayed.

Makefile

@@ -1,6 +1,5 @@
 ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 SHELL := /bin/bash
-TOX_DIR := .tox
 OS := $(shell uname)
 
 # We separate the OSX X and Linux virtualenvs so we can run in a Docker

conf/st2.conf.sample

@@ -113,6 +113,8 @@ url = None
 [database]
 # Specifies database authentication mechanisms. By default, it use SCRAM-SHA-1 with MongoDB 3.0 and later, MONGODB-CR (MongoDB Challenge Response protocol) for older servers.
 authentication_mechanism = None
+# Comma delimited string of compression algorithms to use for transport level compression. Actual algorithm will then be decided based on the algorithms supported by the client and the server. For example: zstd. Defaults to no compression. Keep in mind that zstd is only supported with MongoDB 4.2 and later.
+compressors =
 # Connection retry backoff max (seconds).
 connection_retry_backoff_max_s = 10
 # Backoff multiplier (seconds).
@@ -143,6 +145,8 @@ ssl_keyfile = None
 ssl_match_hostname = True
 # username for db login
 username = None
+# Compression level when compressors is set to zlib. Valid calues are -1 to 9. Defaults to 6.
+zlib_compression_level =
 
 [exporter]
 # Directory to dump data to.

contrib/linux/actions/service.py

@@ -15,6 +15,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+NOTE: This script file utilizes remote-shell-script runner which means it copied as-is to the
+remote host and executed using Python binary available on that systems.
+
+This means it doesn't use pack or StackStorm specific virtual environment which means we can't
+rely on any 3rd party dependencies.
+"""
+
 import re
 import sys
 import os
@@ -23,13 +31,33 @@
 
 from st2common.util.shell import quote_unix
 
-distro = platform.linux_distribution()[0]
+
+def get_linux_distribution():
+    # platform.linux_distribution() is not available in Python >= 3.8
+    if hasattr(platform, "linux_distribution"):
+        distro = platform.linux_distribution()[0]  # pylint: disable=no-member
+    else:
+        # Fall back to shelling out to lsb_release
+        result = subprocess.run(
+            "lsb_release -i -s", shell=True, check=True, stdout=subprocess.PIPE
+        )
+        distro = result.stdout.decode("utf-8").strip()
+
+    if not distro:
+        raise ValueError("Fail to detect distribution we are running on")
+
+    return distro
+
 
 if len(sys.argv) < 3:
     raise ValueError("Usage: service.py <action> <service>")
 
+distro = get_linux_distribution()
+
 args = {"act": quote_unix(sys.argv[1]), "service": quote_unix(sys.argv[2])}
 
+print("Detected distro: %s" % (distro))
+
 if re.search(distro, "Ubuntu"):
     if os.path.isfile("/etc/init/%s.conf" % args["service"]):
         cmd_args = ["service", args["service"], args["act"]]

contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py

@@ -21,6 +21,10 @@
 import mock
 import tempfile
 
+from st2common.util.monkey_patch import use_select_poll_workaround
+
+use_select_poll_workaround()
+
 from oslo_config import cfg
 
 from python_runner import python_runner

fixed-requirements.txt

@@ -31,6 +31,7 @@ prance==0.9.0
 prompt-toolkit==1.0.15
 pyinotify==0.9.6; platform_system=="Linux"
 pymongo==3.11.3
+zstandard==0.15.2
 python-editor==1.0.4
 python-keyczar==0.716
 pytz==2021.1
@@ -42,6 +43,10 @@ retrying==1.3.3
 routes==2.4.1
 semver==2.9.0
 six==1.13.0
+argparse==1.12.2
+argcomplete==1.12.2
+prettytable==2.1.0
+importlib-metadata==3.10.1
 # NOTE: sseclient has various issues which sometimes hang the connection for a long time, etc.
 sseclient-py==1.7
 stevedore==1.30.1

requirements.txt

@@ -8,7 +8,7 @@
 RandomWords
 amqp==2.5.2
 apscheduler==3.7.0
-argcomplete
+argcomplete==1.12.2
 bcrypt==3.2.0
 chardet<3.1.0
 cryptography==3.4.7
@@ -23,6 +23,7 @@ git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backe
 gitpython==2.1.15
 greenlet==1.0.0
 gunicorn==20.1.0
+importlib-metadata==3.10.1
 jinja2==2.11.3
 jsonpath-rw==1.4.0
 jsonschema==2.6.0
@@ -39,7 +40,7 @@ oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
 paramiko==2.7.2
 passlib==1.7.4
-prettytable
+prettytable==2.1.0
 prompt-toolkit==1.0.15
 psutil==5.8.0
 pyinotify==0.9.6; platform_system=="Linux"
@@ -68,3 +69,4 @@ unittest2
 webob==1.8.7
 webtest
 zake==0.2.2
+zstandard==0.15.2

st2actions/tests/unit/test_scheduler.py

@@ -122,7 +122,7 @@ def test_next_execution(self):
         self.reset()
 
         schedule_q_dbs = []
-        delays = [2000, 5000, 4000]
+        delays = [500, 1000, 800]
         expected_order = [0, 2, 1]
         test_cases = []
 
@@ -151,7 +151,7 @@ def test_next_execution(self):
             )
 
         # Wait maximum delay seconds so the query works as expected
-        eventlet.sleep(3.2)
+        eventlet.sleep(1.0)
 
         for index in expected_order:
             test_case = test_cases[index]

st2api/st2api/controllers/v1/packs.py

@@ -186,7 +186,8 @@ def post(self, pack_register_request):
             result["policy_types"] = policies_registrar.register_policy_types(st2common)
 
         use_pack_cache = False
-
+        # TODO: To speed up this operation since it's mostli IO bound we could use green thread
+        # pool here and register different resources concurrently
         fail_on_failure = getattr(pack_register_request, "fail_on_failure", True)
         for type, (Registrar, name) in six.iteritems(ENTITIES):
             if type in types or name in types:

st2api/tests/unit/controllers/v1/test_executions_auth.py

@@ -130,7 +130,7 @@ class ActionExecutionControllerTestCaseAuthEnabled(FunctionalTest):
 
     @classmethod
     @mock.patch.object(Token, "get", mock.MagicMock(side_effect=mock_get_token))
-    @mock.patch.object(User, "get_by_name", mock_get_by_name)
+    @mock.patch.object(User, "get_by_name", mock.MagicMock(return_value=TEST_USER))
     @mock.patch.object(
         action_validator, "validate_action", mock.MagicMock(return_value=True)
     )

st2api/tests/unit/controllers/v1/test_packs.py

@@ -17,6 +17,7 @@
 
 import requests
 import mock
+import sys
 
 from st2common.content.loader import ContentPackLoader
 from st2common.models.db.pack import PackDB
@@ -345,6 +346,13 @@ def test_index_health(self):
     def test_index_health_broken(self):
         resp = self.app.get("/v1/packs/index/health")
 
+        # up to Py3.6 requests.exceptions.RequestException() appends a trailing ,
+        if sys.version_info < (3, 8):
+            broken_index_message = "RequestException('index is broken',)"
+        else:
+            broken_index_message = "RequestException('index is broken')"
+
+        self.maxDiff = None
         self.assertEqual(resp.status_int, 200)
         self.assertEqual(
             resp.json,
@@ -361,7 +369,7 @@ def test_index_health_broken(self):
                         },
                         {
                             "url": "http://broken.example.com",
-                            "message": "RequestException('index is broken',)",
+                            "message": broken_index_message,
                             "packs": 0,
                             "error": "unresponsive",
                         },
@@ -494,8 +502,6 @@ def test_packs_register_endpoint(self, mock_get_packs):
         packs_base_path = os.path.join(fixtures_base_path, "packs")
         pack_names = [
             "dummy_pack_1",
-            "dummy_pack_2",
-            "dummy_pack_3",
             "dummy_pack_10",
         ]
         mock_return_value = {}

st2client/in-requirements.txt

@@ -1,4 +1,5 @@
 # Remember to list implicit packages here, otherwise version won't be fixated!
+importlib-metadata
 argcomplete
 prettytable
 pytz