From c6c1a2ef171f10bf51ebc92532fe88ce411b9b9c Mon Sep 17 00:00:00 2001 From: Carlos Date: Wed, 3 Mar 2021 16:16:02 +0100 Subject: [PATCH 01/58] WIP - start of work - Bump eventlet/greenlet to latest to benefit from threading fix for python3.7+ - Bump pymongo/mongoengine to support Mongo4.4 which is the only version packaged for Ubuntu 20.04. - Added Python3.8 into tox tests. --- fixed-requirements.txt | 8 ++-- requirements.txt | 8 ++-- st2actions/requirements.txt | 2 +- st2api/requirements.txt | 6 +-- st2auth/requirements.txt | 4 +- st2common/requirements.txt | 8 ++-- st2exporter/requirements.txt | 2 +- st2reactor/requirements.txt | 2 +- st2stream/requirements.txt | 6 +-- tox.ini | 79 ++++++++++++++++++++++++++++++++++-- 10 files changed, 99 insertions(+), 26 deletions(-) diff --git a/fixed-requirements.txt b/fixed-requirements.txt index a98a90dc34..5d121d6ae5 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -9,17 +9,17 @@ dnspython>=1.16.0,<2.0.0 cryptography==3.3.2 # Note: 0.20.0 removed select.poll() on which some of our code and libraries we # depend on rely -eventlet==0.25.1 +eventlet==0.30.1 flex==6.14.0 gitpython==2.1.15 # Note: greenlet is used by eventlet -greenlet==0.4.15 +greenlet==0.4.17 gunicorn==19.9.0 jsonpath-rw==1.4.0 jsonschema==2.6.0 kombu==4.6.6 lockfile==0.12.2 -mongoengine==0.18.2 +mongoengine==0.22.1 networkx==1.11 # NOTE: Recent version substantially affect the performance and add big import time overhead # See https://github.com/StackStorm/st2/issues/4160#issuecomment-394386433 for details @@ -30,7 +30,7 @@ passlib==1.7.1 prance==0.9.0 prompt-toolkit==1.0.15 pyinotify==0.9.6; platform_system=="Linux" -pymongo==3.10.0 +pymongo==3.11.3 python-editor==1.0.4 python-gnupg==0.4.5 python-keyczar==0.716 diff --git a/requirements.txt b/requirements.txt index 2b5aa2436c..010ae18db0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ bcrypt==3.1.7 chardet<3.1.0 cryptography==3.3.2 dnspython<2.0.0,>=1.16.0 -eventlet==0.25.1 +eventlet==0.30.1 flex==6.14.0 git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper git+https://github.com/StackStorm/orquesta.git@v1.3.0#egg=orquesta @@ -21,7 +21,7 @@ git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2- git+https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend gitpython==2.1.15 -greenlet==0.4.15 +greenlet==0.4.17 gunicorn==19.9.0 ipaddr jinja2==2.10.3 @@ -30,7 +30,7 @@ jsonschema==2.6.0 kombu==4.6.6 lockfile==0.12.2 mock==2.0.0 -mongoengine==0.18.2 +mongoengine==0.22.1 more-itertools==5.0.0 networkx==1.11 nose @@ -44,7 +44,7 @@ prettytable prompt-toolkit==1.0.15 psutil==5.6.6 pyinotify==0.9.6 ; platform_system == "Linux" -pymongo==3.10.0 +pymongo==3.11.3 pyrabbit python-dateutil==2.8.0 python-editor==1.0.4 diff --git a/st2actions/requirements.txt b/st2actions/requirements.txt index 2ef4e057a8..c51745069d 100755 --- a/st2actions/requirements.txt +++ b/st2actions/requirements.txt @@ -7,7 +7,7 @@ # update the component requirements.txt apscheduler==3.6.3 chardet<3.1.0 -eventlet==0.25.1 +eventlet==0.30.1 git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper gitpython==2.1.15 jinja2==2.10.3 diff --git a/st2api/requirements.txt b/st2api/requirements.txt index e9f8dca18f..79d83ceee0 100644 --- a/st2api/requirements.txt +++ b/st2api/requirements.txt @@ -5,13 +5,13 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -eventlet==0.25.1 +eventlet==0.30.1 gunicorn==19.9.0 jsonschema==2.6.0 kombu==4.6.6 -mongoengine==0.18.2 +mongoengine==0.22.1 oslo.config<1.13,>=1.12.1 oslo.utils<5.0,>=4.0.0 -pymongo==3.10.0 +pymongo==3.11.3 simplejson six==1.13.0 diff --git a/st2auth/requirements.txt b/st2auth/requirements.txt index 838f0f9918..cc3f0fc95f 100644 --- a/st2auth/requirements.txt +++ b/st2auth/requirements.txt @@ -6,12 +6,12 @@ # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt bcrypt==3.1.7 -eventlet==0.25.1 +eventlet==0.30.1 git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file git+https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap gunicorn==19.9.0 oslo.config<1.13,>=1.12.1 passlib==1.7.1 -pymongo==3.10.0 +pymongo==3.11.3 six==1.13.0 stevedore==1.30.1 diff --git a/st2common/requirements.txt b/st2common/requirements.txt index c3ee24e5f3..0777afa042 100644 --- a/st2common/requirements.txt +++ b/st2common/requirements.txt @@ -10,23 +10,23 @@ apscheduler==3.6.3 chardet<3.1.0 cryptography==3.3.2 dnspython<2.0.0,>=1.16.0 -eventlet==0.25.1 +eventlet==0.30.1 flex==6.14.0 git+https://github.com/StackStorm/orquesta.git@v1.3.0#egg=orquesta git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend gitpython==2.1.15 -greenlet==0.4.15 +greenlet==0.4.17 ipaddr jinja2==2.10.3 jsonpath-rw==1.4.0 jsonschema==2.6.0 kombu==4.6.6 lockfile==0.12.2 -mongoengine==0.18.2 +mongoengine==0.22.1 networkx==1.11 oslo.config<1.13,>=1.12.1 paramiko==2.7.1 -pymongo==3.10.0 +pymongo==3.11.3 python-dateutil==2.8.0 python-statsd==2.1.0 pyyaml==5.1.2 diff --git a/st2exporter/requirements.txt b/st2exporter/requirements.txt index 21cb97016f..f5a7e113cc 100644 --- a/st2exporter/requirements.txt +++ b/st2exporter/requirements.txt @@ -5,7 +5,7 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -eventlet==0.25.1 +eventlet==0.30.1 kombu==4.6.6 oslo.config<1.13,>=1.12.1 six==1.13.0 diff --git a/st2reactor/requirements.txt b/st2reactor/requirements.txt index 3da786fd47..805b51f943 100644 --- a/st2reactor/requirements.txt +++ b/st2reactor/requirements.txt @@ -6,7 +6,7 @@ # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt apscheduler==3.6.3 -eventlet==0.25.1 +eventlet==0.30.1 jsonpath-rw==1.4.0 jsonschema==2.6.0 kombu==4.6.6 diff --git a/st2stream/requirements.txt b/st2stream/requirements.txt index fe3cbd71d7..e20ca20ad9 100644 --- a/st2stream/requirements.txt +++ b/st2stream/requirements.txt @@ -5,12 +5,12 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -eventlet==0.25.1 +eventlet==0.30.1 gunicorn==19.9.0 jsonschema==2.6.0 kombu==4.6.6 -mongoengine==0.18.2 +mongoengine==0.22.1 oslo.config<1.13,>=1.12.1 oslo.utils<5.0,>=4.0.0 -pymongo==3.10.0 +pymongo==3.11.3 six==1.13.0 diff --git a/tox.ini b/tox.ini index 451ceee8e1..5ec83d0e87 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36-unit,py36-integration +envlist = py36-unit,py36-integration.py38-unit,py38-integration skipsdist = true skip_missing_interpreters=true @@ -18,7 +18,7 @@ deps = -r{toxinidir}/requirements.txt -e{toxinidir}/st2common -e{toxinidir}/st2reactor -# Python 3 tasks +# Python 3.6 tasks [testenv:py36-unit] basepython = python3.6 setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner:{toxinidir}/st2common/tests/runners/mock_query_callback @@ -71,7 +71,80 @@ commands = [testenv:py36-integration] basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2auth:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2auth:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner + VIRTUALENV_DIR = {envdir} +passenv = NOSE_WITH_TIMER TRAVIS ST2_CI +install_command = pip install -U --force-reinstall {opts} {packages} +deps = virtualenv + -r{toxinidir}/requirements.txt + -e{toxinidir}/st2client + -e{toxinidir}/st2common +commands = + nosetests --rednose --immediate -sv st2actions/tests/integration/ + nosetests --rednose --immediate -sv st2api/tests/integration/ + nosetests --rednose --immediate -sv st2common/tests/integration/ + nosetests --rednose --immediate -sv st2exporter/tests/integration/ + nosetests --rednose --immediate -sv st2reactor/tests/integration/ + nosetests --rednose --immediate -sv contrib/runners/action_chain_runner/tests/integration/ + nosetests --rednose --immediate -sv contrib/runners/local_runner/tests/integration/ + nosetests --rednose --immediate -sv contrib/runners/orquesta_runner/tests/integration/ + nosetests --rednose --immediate -sv st2tests/integration/orquesta/ + nosetests --rednose --immediate -sv contrib/runners/python_runner/tests/integration/ + +# Python 3.8 tasks +[testenv:py38-unit] +basepython = python3.8 +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner:{toxinidir}/st2common/tests/runners/mock_query_callback + VIRTUALENV_DIR = {envdir} +passenv = NOSE_WITH_TIMER TRAVIS ST2_CI +install_command = pip install -U --force-reinstall {opts} {packages} +deps = virtualenv + -r{toxinidir}/requirements.txt + -e{toxinidir}/st2client + -e{toxinidir}/st2common + -e{toxinidir}/st2auth +commands = + nosetests --rednose --immediate -sv st2actions/tests/unit/ + nosetests --rednose --immediate -sv st2auth/tests/unit/ + nosetests --rednose --immediate -sv st2api/tests/unit/controllers/v1/ + nosetests --rednose --immediate -sv st2api/tests/unit/controllers/exp/ + nosetests --rednose --immediate -sv st2common/tests/unit/ + nosetests --rednose --immediate -sv st2client/tests/unit/ + nosetests --rednose --immediate -sv st2exporter/tests/unit/ + nosetests --rednose --immediate -sv st2reactor/tests/unit/ + nosetests --rednose --immediate -sv st2stream/tests/unit/ + nosetests --rednose --immediate -sv contrib/runners/action_chain_runner/tests/unit/ + nosetests --rednose --immediate -sv contrib/runners/inquirer_runner/tests/unit/ + nosetests --rednose --immediate -sv contrib/runners/announcement_runner/tests/unit/ + nosetests --rednose --immediate -sv contrib/runners/http_runner/tests/unit/ + nosetests --rednose --immediate -sv contrib/runners/noop_runner/tests/unit/ + nosetests --rednose --immediate -sv contrib/runners/local_runner/tests/unit/ + nosetests --rednose --immediate -sv contrib/runners/orquesta_runner/tests/unit/ + nosetests --rednose --immediate -sv contrib/runners/python_runner/tests/unit/ + nosetests --rednose --immediate -sv contrib/runners/winrm_runner/tests/unit/ + +[testenv:py38-packs] +basepython = python3.8 +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner + VIRTUALENV_DIR = {envdir} +passenv = NOSE_WITH_TIMER TRAVIS ST2_CI +install_command = pip install -U --force-reinstall {opts} {packages} +deps = virtualenv + -r{toxinidir}/requirements.txt + -e{toxinidir}/st2client + -e{toxinidir}/st2common +commands = + st2-run-pack-tests -c -t -x -p contrib/packs + st2-run-pack-tests -c -t -x -p contrib/core + st2-run-pack-tests -c -t -x -p contrib/default + st2-run-pack-tests -c -t -x -p contrib/chatops + st2-run-pack-tests -c -t -x -p contrib/examples + st2-run-pack-tests -c -t -x -p contrib/linux + st2-run-pack-tests -c -t -x -p contrib/hello_st2 + +[testenv:py38-integration] +basepython = python3.8 +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2auth:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner VIRTUALENV_DIR = {envdir} passenv = NOSE_WITH_TIMER TRAVIS ST2_CI install_command = pip install -U --force-reinstall {opts} {packages} From 5fbd67411316bd15a3fcd913e1c9d4a3d6e09d8f Mon Sep 17 00:00:00 2001 From: Carlos Date: Wed, 3 Mar 2021 17:13:43 +0100 Subject: [PATCH 02/58] Add python3.8 to github actions --- .github/workflows/ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 56b30edac8..57ab9613d9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -30,16 +30,16 @@ jobs: include: - name: 'Lint Checks' task: 'ci-checks' - python-version: '3.6' + python-version: [3.6, 3.8] - name: 'Compile' task: 'ci-compile' - python-version: '3.6' + python-version: [3.6, 3.8] - name: 'Pack Tests' task: 'ci-packs-tests' - python-version: '3.6' + python-version: [3.6, 3.8] - name: 'Unit Tests' task: 'ci-unit' - python-version: '3.6' + python-version: [3.6, 3.8] # Integration tests are not working yet, still done in Travis # - name: 'Integration Tests' # task: 'ci-integration' From b14d3a725cda555d72a78a4723fe49ed898b42d4 Mon Sep 17 00:00:00 2001 From: Carlos Date: Wed, 3 Mar 2021 17:20:56 +0100 Subject: [PATCH 03/58] Move python versions to be a subkey of matrix. --- .github/workflows/ci.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 57ab9613d9..0ae3d9eee7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,19 +27,16 @@ jobs: strategy: fail-fast: false matrix: + python-version: [3.6, 3.8] include: - name: 'Lint Checks' task: 'ci-checks' - python-version: [3.6, 3.8] - name: 'Compile' task: 'ci-compile' - python-version: [3.6, 3.8] - name: 'Pack Tests' task: 'ci-packs-tests' - python-version: [3.6, 3.8] - name: 'Unit Tests' task: 'ci-unit' - python-version: [3.6, 3.8] # Integration tests are not working yet, still done in Travis # - name: 'Integration Tests' # task: 'ci-integration' From e2a1b403076a37fe45307612113d55053c2a73c2 Mon Sep 17 00:00:00 2001 From: Carlos Date: Thu, 4 Mar 2021 01:11:01 +0100 Subject: [PATCH 04/58] Use mongo4.4 for ci testing and try disabling monkey patching which caused unit tests to halt. --- .github/workflows/ci.yaml | 2 +- st2common/st2common/util/monkey_patch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0ae3d9eee7..4979c02d87 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,7 @@ jobs: # task: 'ci-integration' services: mongo: - image: mongo:4.0 + image: mongo:4.4 ports: - 27017:27017 # Can't use RabbitMQ here for Integrations because we rely on custom config diff --git a/st2common/st2common/util/monkey_patch.py b/st2common/st2common/util/monkey_patch.py index 5a042fd656..4e7057582d 100644 --- a/st2common/st2common/util/monkey_patch.py +++ b/st2common/st2common/util/monkey_patch.py @@ -31,7 +31,7 @@ PARENT_ARGS_FLAG = '--parent-args=' -def monkey_patch(patch_thread=None): +def monkey_patch(patch_thread=False): """ Function which performs eventlet monkey patching and also takes into account "--use-debugger" argument in the command line arguments. From 0133f302978b8d60159230fe7b251fa4ffb77dfe Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Sat, 6 Mar 2021 00:41:13 +0100 Subject: [PATCH 05/58] Move away from positional arguments on UserDB calls for the rules and aliasexecution controllers --- st2api/st2api/controllers/v1/aliasexecution.py | 2 +- st2api/st2api/controllers/v1/rules.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/st2api/st2api/controllers/v1/aliasexecution.py b/st2api/st2api/controllers/v1/aliasexecution.py index 7ecc14d62e..17c4ec73d6 100644 --- a/st2api/st2api/controllers/v1/aliasexecution.py +++ b/st2api/st2api/controllers/v1/aliasexecution.py @@ -113,7 +113,7 @@ def _post(self, payload, requester_user, show_secrets=False, match_multiple=Fals return if not requester_user: - requester_user = UserDB(cfg.CONF.system_user.user) + requester_user = UserDB(name=cfg.CONF.system_user.user) format_str = payload.format or '' command = payload.command or '' diff --git a/st2api/st2api/controllers/v1/rules.py b/st2api/st2api/controllers/v1/rules.py index 5904f9140e..4f0a47e7ac 100644 --- a/st2api/st2api/controllers/v1/rules.py +++ b/st2api/st2api/controllers/v1/rules.py @@ -109,7 +109,7 @@ def post(self, rule, requester_user): permission_type=permission_type) if not requester_user: - requester_user = UserDB(cfg.CONF.system_user.user) + requester_user = UserDB(name=cfg.CONF.system_user.user) # Validate that the authenticated user is admin if user query param is provided user = requester_user.name From 75579b5b0391a983ea3079d0ae810720ad083f74 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Sat, 6 Mar 2021 00:47:27 +0100 Subject: [PATCH 06/58] Move away from positional arguments on UserDB calls for the actionexecutions controller --- st2api/st2api/controllers/v1/actionexecutions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/st2api/st2api/controllers/v1/actionexecutions.py b/st2api/st2api/controllers/v1/actionexecutions.py index b0aa4e9e1d..28d6e7305b 100644 --- a/st2api/st2api/controllers/v1/actionexecutions.py +++ b/st2api/st2api/controllers/v1/actionexecutions.py @@ -110,7 +110,7 @@ def _handle_schedule_execution(self, liveaction_api, requester_user, context_str :type liveaction: :class:`LiveActionAPI` """ if not requester_user: - requester_user = UserDB(cfg.CONF.system_user.user) + requester_user = UserDB(name=cfg.CONF.system_user.user) # Assert action ref is valid action_ref = liveaction_api.action @@ -575,7 +575,7 @@ def put(self, id, liveaction_api, requester_user, show_secrets=False): """ if not requester_user: - requester_user = UserDB(cfg.CONF.system_user.user) + requester_user = UserDB(name=cfg.CONF.system_user.user) from_model_kwargs = { 'mask_secrets': self._get_mask_secrets(requester_user, show_secrets=show_secrets) @@ -660,7 +660,7 @@ def delete(self, id, requester_user, show_secrets=False): """ if not requester_user: - requester_user = UserDB(cfg.CONF.system_user.user) + requester_user = UserDB(name=cfg.CONF.system_user.user) from_model_kwargs = { 'mask_secrets': self._get_mask_secrets(requester_user, show_secrets=show_secrets) From 184611b142c4f4eaca957bd0912671cad91ab264 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Sat, 6 Mar 2021 00:48:59 +0100 Subject: [PATCH 07/58] Move away from positional arguments on UserDB calls for the keyvalue controller --- st2api/st2api/controllers/v1/keyvalue.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/st2api/st2api/controllers/v1/keyvalue.py b/st2api/st2api/controllers/v1/keyvalue.py index eab8cb025a..bc1c687aac 100644 --- a/st2api/st2api/controllers/v1/keyvalue.py +++ b/st2api/st2api/controllers/v1/keyvalue.py @@ -78,7 +78,7 @@ def get_one(self, name, requester_user, scope=FULL_SYSTEM_SCOPE, user=None, decr scope = FULL_USER_SCOPE if not requester_user: - requester_user = UserDB(cfg.CONF.system_user.user) + requester_user = UserDB(name=cfg.CONF.system_user.user) scope = get_datastore_full_scope(scope) self._validate_scope(scope=scope) @@ -144,7 +144,7 @@ def get_all(self, requester_user, prefix=None, scope=FULL_SYSTEM_SCOPE, user=Non scope = FULL_USER_SCOPE if not requester_user: - requester_user = UserDB(cfg.CONF.system_user.user) + requester_user = UserDB(name=cfg.CONF.system_user.user) scope = get_datastore_full_scope(scope) @@ -264,7 +264,7 @@ def put(self, kvp, name, requester_user, scope=FULL_SYSTEM_SCOPE): scope = FULL_SYSTEM_SCOPE if not requester_user: - requester_user = UserDB(cfg.CONF.system_user.user) + requester_user = UserDB(name=cfg.CONF.system_user.user) scope = getattr(kvp, 'scope', scope) scope = get_datastore_full_scope(scope) @@ -342,7 +342,7 @@ def delete(self, name, requester_user, scope=FULL_SYSTEM_SCOPE, user=None): scope = FULL_SYSTEM_SCOPE if not requester_user: - requester_user = UserDB(cfg.CONF.system_user.user) + requester_user = UserDB(name=cfg.CONF.system_user.user) scope = get_datastore_full_scope(scope) self._validate_scope(scope=scope) From 20b4a6302b114bcf059d76a4622b9c6f1cf950d0 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Sat, 6 Mar 2021 00:49:42 +0100 Subject: [PATCH 08/58] Move away from positional arguments on UserDB calls for the packs controller --- st2api/st2api/controllers/v1/packs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2api/st2api/controllers/v1/packs.py b/st2api/st2api/controllers/v1/packs.py index 6193a3f01f..8d19da5b5e 100644 --- a/st2api/st2api/controllers/v1/packs.py +++ b/st2api/st2api/controllers/v1/packs.py @@ -107,7 +107,7 @@ def post(self, pack_install_request, requester_user=None): parameters['skip_dependencies'] = True if not requester_user: - requester_user = UserDB(cfg.CONF.system_user.user) + requester_user = UserDB(name=cfg.CONF.system_user.user) new_liveaction_api = LiveActionCreateAPI(action='packs.install', parameters=parameters, @@ -134,7 +134,7 @@ def post(self, pack_uninstall_request, ref_or_id=None, requester_user=None): } if not requester_user: - requester_user = UserDB(cfg.CONF.system_user.user) + requester_user = UserDB(name=cfg.CONF.system_user.user) new_liveaction_api = LiveActionCreateAPI(action='packs.uninstall', parameters=parameters, From e1a595a3b7c0f8bdc3cd9dbd032d1bdc8b91db45 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Sat, 6 Mar 2021 00:50:12 +0100 Subject: [PATCH 09/58] Move away from positional arguments on UserDB calls for the rules controller (this time for real) --- st2api/st2api/controllers/v1/rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2api/st2api/controllers/v1/rules.py b/st2api/st2api/controllers/v1/rules.py index 4f0a47e7ac..f3cae5c678 100644 --- a/st2api/st2api/controllers/v1/rules.py +++ b/st2api/st2api/controllers/v1/rules.py @@ -168,7 +168,7 @@ def put(self, rule, rule_ref_or_id, requester_user): LOG.debug('PUT /rules/ lookup with id=%s found object: %s', rule_ref_or_id, rule_db) if not requester_user: - requester_user = UserDB(cfg.CONF.system_user.user) + requester_user = UserDB(name=cfg.CONF.system_user.user) # Validate that the authenticated user is admin if user query param is provided user = requester_user.name rbac_utils.assert_user_is_admin_if_user_query_param_is_provided(user_db=requester_user, From 645bad7b95e4d50d9fba3ef14f07b742ec776591 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Sat, 6 Mar 2021 00:56:33 +0100 Subject: [PATCH 10/58] Move away from positional arguments on UserDB calls for the inquiry service --- st2common/st2common/services/inquiry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/services/inquiry.py b/st2common/st2common/services/inquiry.py index 5b511b3a97..6a942b040e 100644 --- a/st2common/st2common/services/inquiry.py +++ b/st2common/st2common/services/inquiry.py @@ -59,7 +59,7 @@ def check_inquiry(inquiry): def check_permission(inquiry, requester): # Normalize user object. user_db = ( - auth_db_models.UserDB(requester) + auth_db_models.UserDB(name=requester) if isinstance(requester, six.string_types) else requester ) From c4bbe78253c1413a9dfe57dafe05309aa44bd15d Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Sat, 6 Mar 2021 01:01:25 +0100 Subject: [PATCH 11/58] Update User.get_by_name calls to use named arguments instead of the deprecated positional ones --- st2common/st2common/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/router.py b/st2common/st2common/router.py index 29b34031b4..af99f595be 100644 --- a/st2common/st2common/router.py +++ b/st2common/st2common/router.py @@ -292,7 +292,7 @@ def __call__(self, req): elif 'api-key' in definition['name'].lower(): auth_method = 'API key' - context['user'] = User.get_by_name(auth_resp.user) + context['user'] = User.get_by_name(name=auth_resp.user) context['auth_info'] = { 'method': auth_method, 'location': definition['in'] From d74f2a58ac01ceb62ab4de0d15aae6365e0e483e Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Sat, 6 Mar 2021 01:05:05 +0100 Subject: [PATCH 12/58] Revert "Update User.get_by_name calls to use named arguments instead of the deprecated positional ones" This reverts commit c4bbe78253c1413a9dfe57dafe05309aa44bd15d as it did not fix the issue --- st2common/st2common/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/router.py b/st2common/st2common/router.py index af99f595be..29b34031b4 100644 --- a/st2common/st2common/router.py +++ b/st2common/st2common/router.py @@ -292,7 +292,7 @@ def __call__(self, req): elif 'api-key' in definition['name'].lower(): auth_method = 'API key' - context['user'] = User.get_by_name(name=auth_resp.user) + context['user'] = User.get_by_name(auth_resp.user) context['auth_info'] = { 'method': auth_method, 'location': definition['in'] From a16c243a789292549d43798997d3aa68cfd0e3e2 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Mon, 8 Mar 2021 21:59:52 +0000 Subject: [PATCH 13/58] Switch from side_effect to return_value to fix test_executions_auth tests --- st2api/tests/unit/controllers/v1/test_executions_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2api/tests/unit/controllers/v1/test_executions_auth.py b/st2api/tests/unit/controllers/v1/test_executions_auth.py index f1045a7d54..7a7f02880b 100644 --- a/st2api/tests/unit/controllers/v1/test_executions_auth.py +++ b/st2api/tests/unit/controllers/v1/test_executions_auth.py @@ -126,7 +126,7 @@ class ActionExecutionControllerTestCaseAuthEnabled(FunctionalTest): @classmethod @mock.patch.object(Token, "get", mock.MagicMock(side_effect=mock_get_token)) - @mock.patch.object(User, "get_by_name", mock.MagicMock(side_effect=UserDB)) + @mock.patch.object(User, "get_by_name", mock.MagicMock(return_value=TEST_USER)) @mock.patch.object( action_validator, "validate_action", mock.MagicMock(return_value=True) ) From 804d7172a5fe5459ece3c5caf2d1de5fbb20a0b0 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Mon, 8 Mar 2021 22:00:37 +0000 Subject: [PATCH 14/58] Update 2 more UserDB calls in the keyvalue utility and the inquiries garbage collector in st2common to use named instead of positional parameters --- st2common/st2common/garbage_collection/inquiries.py | 2 +- st2common/st2common/util/keyvalue.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/st2common/st2common/garbage_collection/inquiries.py b/st2common/st2common/garbage_collection/inquiries.py index ad95126b21..2381472a73 100644 --- a/st2common/st2common/garbage_collection/inquiries.py +++ b/st2common/st2common/garbage_collection/inquiries.py @@ -90,7 +90,7 @@ def purge_inquiries(logger): # Request that root workflow resumes root_liveaction = action_service.get_root_liveaction(liveaction_db) action_service.request_resume( - root_liveaction, UserDB(cfg.CONF.system_user.user) + root_liveaction, UserDB(name=cfg.CONF.system_user.user) ) logger.info('Marked %s ttl-expired Inquiries as "timed out".' % (gc_count)) diff --git a/st2common/st2common/util/keyvalue.py b/st2common/st2common/util/keyvalue.py index cad32250a8..1fc538025f 100644 --- a/st2common/st2common/util/keyvalue.py +++ b/st2common/st2common/util/keyvalue.py @@ -94,7 +94,7 @@ def get_key(key=None, user_db=None, scope=None, decrypt=False): if not user_db: # Use system user - user_db = UserDB(cfg.CONF.system_user.user) + user_db = UserDB(name=cfg.CONF.system_user.user) scope, key_id = _derive_scope_and_key(key=key, user=user_db.name, scope=scope) scope = get_datastore_full_scope(scope) From ff87e0833264f9b36f9ca22153a8e291846ea6b2 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 11 Mar 2021 23:10:30 +0000 Subject: [PATCH 15/58] Drop the deprecated format parameter von queryset.explain call for the queryset profiling It's deprecated since https://github.com/MongoEngine/mongoengine/pull/2113 --- st2common/st2common/models/utils/profiling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/models/utils/profiling.py b/st2common/st2common/models/utils/profiling.py index 47add2adc3..55d6b486c7 100644 --- a/st2common/st2common/models/utils/profiling.py +++ b/st2common/st2common/models/utils/profiling.py @@ -83,7 +83,7 @@ def log_query_and_profile_data_for_queryset(queryset): # Note: We need to clone the queryset when using explain because explain advances the cursor # internally which changes the function result cloned_queryset = queryset.clone() - explain_info = cloned_queryset.explain(format=True) + explain_info = cloned_queryset.explain() if mongo_query is not None and collection_name is not None: mongo_shell_query = construct_mongo_shell_query( From 2f249ad5987ecf71aded18069f17da693ae10656 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 11 Mar 2021 23:20:07 +0000 Subject: [PATCH 16/58] Rename MongoEngineConnectionError by ConnectionFailure This is required due to https://github.com/MongoEngine/mongoengine/pull/2111/files --- st2common/st2common/exceptions/workflow.py | 2 +- st2common/st2common/persistence/db_init.py | 2 +- .../tests/unit/services/test_workflow_service_retries.py | 8 ++++---- st2common/tests/unit/test_exceptions_workflow.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/st2common/st2common/exceptions/workflow.py b/st2common/st2common/exceptions/workflow.py index 2a346819be..370030c6ec 100644 --- a/st2common/st2common/exceptions/workflow.py +++ b/st2common/st2common/exceptions/workflow.py @@ -30,7 +30,7 @@ def retry_on_connection_errors(exc): LOG.warning("Determining if exception %s should be retried.", type(exc)) retrying = isinstance(exc, tooz.coordination.ToozConnectionError) or isinstance( - exc, mongoengine.connection.MongoEngineConnectionError + exc, mongoengine.connection.ConnectionFailure ) if retrying: diff --git a/st2common/st2common/persistence/db_init.py b/st2common/st2common/persistence/db_init.py index 678ca71ccd..f43e2fd06b 100644 --- a/st2common/st2common/persistence/db_init.py +++ b/st2common/st2common/persistence/db_init.py @@ -35,7 +35,7 @@ def _retry_if_connection_error(error): # If this does become an issue look for "Cannot connect to database" at the # start of error msg. is_connection_error = isinstance( - error, mongoengine.connection.MongoEngineConnectionError + error, mongoengine.connection.ConnectionFailure ) if is_connection_error: LOG.warn("Retry on ConnectionError - %s", error) diff --git a/st2common/tests/unit/services/test_workflow_service_retries.py b/st2common/tests/unit/services/test_workflow_service_retries.py index baa79c6954..f39315a308 100644 --- a/st2common/tests/unit/services/test_workflow_service_retries.py +++ b/st2common/tests/unit/services/test_workflow_service_retries.py @@ -196,8 +196,8 @@ def test_retries_exhausted_from_coordinator_connection_error(self): "update_task_state", mock.MagicMock( side_effect=[ - mongoengine.connection.MongoEngineConnectionError(), - mongoengine.connection.MongoEngineConnectionError(), + mongoengine.connection.ConnectionFailure(), + mongoengine.connection.ConnectionFailure(), None, ] ), @@ -227,7 +227,7 @@ def test_recover_from_database_connection_error(self): @mock.patch.object( wf_svc, "update_task_state", - mock.MagicMock(side_effect=mongoengine.connection.MongoEngineConnectionError()), + mock.MagicMock(side_effect=mongoengine.connection.ConnectionFailure()), ) def test_retries_exhausted_from_database_connection_error(self): wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") @@ -248,7 +248,7 @@ def test_retries_exhausted_from_database_connection_error(self): # The connection error should raise if retries are exhaused. self.assertRaises( - mongoengine.connection.MongoEngineConnectionError, + mongoengine.connection.ConnectionFailure, wf_svc.handle_action_execution_completion, tk1_ac_ex_db, ) diff --git a/st2common/tests/unit/test_exceptions_workflow.py b/st2common/tests/unit/test_exceptions_workflow.py index a9fbcc549f..fbb6dbd1fe 100644 --- a/st2common/tests/unit/test_exceptions_workflow.py +++ b/st2common/tests/unit/test_exceptions_workflow.py @@ -42,7 +42,7 @@ def test_retry_on_connection_errors(self): exc = coordination.ToozConnectionError("foobar") self.assertTrue(wf_exc.retry_on_connection_errors(exc)) - exc = mongoengine.connection.MongoEngineConnectionError() + exc = mongoengine.connection.ConnectionFailure() self.assertTrue(wf_exc.retry_on_connection_errors(exc)) def test_do_not_retry_on_connection_errors(self): From 796a68bdbeb247f7113e9c418dd6a516b7d1566e Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Sun, 28 Mar 2021 22:54:11 +0200 Subject: [PATCH 17/58] Specify the full loader on yaml.load() to surpress warning --- st2common/tests/unit/test_content_loader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/st2common/tests/unit/test_content_loader.py b/st2common/tests/unit/test_content_loader.py index 566ddf4f76..279d04dc4a 100644 --- a/st2common/tests/unit/test_content_loader.py +++ b/st2common/tests/unit/test_content_loader.py @@ -21,7 +21,7 @@ import yaml -from yaml import SafeLoader +from yaml import SafeLoader, FullLoader try: from yaml import CSafeLoader @@ -121,8 +121,8 @@ def test_yaml_safe_load(self): dumped = yaml.dump(Foo) self.assertTrue("!!python" in dumped) - # Regular load should work, but safe wrapper should fail - result = yaml.load(dumped) + # Regular full load should work, but safe wrapper should fail + result = yaml.load(dumped, Loader=FullLoader) self.assertTrue(result) self.assertRaisesRegexp( From 42daa72f24b35265ccbddde884d313e327a7f3ae Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Sun, 28 Mar 2021 22:04:21 +0000 Subject: [PATCH 18/58] Bump eventlet to 0.30.2 --- fixed-requirements.txt | 2 +- requirements.txt | 4 ++-- st2actions/requirements.txt | 2 +- st2api/requirements.txt | 2 +- st2auth/requirements.txt | 2 +- st2common/requirements.txt | 3 +-- st2exporter/requirements.txt | 2 +- st2reactor/requirements.txt | 2 +- st2stream/requirements.txt | 2 +- 9 files changed, 10 insertions(+), 11 deletions(-) diff --git a/fixed-requirements.txt b/fixed-requirements.txt index 3b1ec2157e..5c82ef84c9 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -9,7 +9,7 @@ dnspython>=1.16.0,<2.0.0 cryptography==3.3.2 # Note: 0.20.0 removed select.poll() on which some of our code and libraries we # depend on rely -eventlet==0.30.1 +eventlet==0.30.2 flex==6.14.0 gitpython==2.1.15 # Note: greenlet is used by eventlet diff --git a/requirements.txt b/requirements.txt index a8802024ba..5c4de60bcf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ bcrypt==3.1.7 chardet<3.1.0 cryptography==3.3.2 dnspython>=1.16.0,<2.0.0 -eventlet==0.30.1 +eventlet==0.30.2 flex==6.14.0 git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper git+https://github.com/StackStorm/orquesta.git@v1.3.0#egg=orquesta @@ -43,7 +43,7 @@ passlib==1.7.1 prettytable prompt-toolkit==1.0.15 psutil==5.6.6 -pyinotify==0.9.6 ; platform_system=="Linux" +pyinotify==0.9.6; platform_system=="Linux" pymongo==3.11.3 pyrabbit python-dateutil==2.8.0 diff --git a/st2actions/requirements.txt b/st2actions/requirements.txt index f3b1f7648f..fbb4d4c49e 100644 --- a/st2actions/requirements.txt +++ b/st2actions/requirements.txt @@ -7,7 +7,7 @@ # update the component requirements.txt apscheduler==3.6.3 chardet<3.1.0 -eventlet==0.30.1 +eventlet==0.30.2 git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper gitpython==2.1.15 jinja2==2.11.3 diff --git a/st2api/requirements.txt b/st2api/requirements.txt index 5ef4e4075b..22a52cc9a6 100644 --- a/st2api/requirements.txt +++ b/st2api/requirements.txt @@ -5,7 +5,7 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -eventlet==0.30.1 +eventlet==0.30.2 gunicorn==19.9.0 jsonschema==2.6.0 kombu==4.6.6 diff --git a/st2auth/requirements.txt b/st2auth/requirements.txt index b70c22f94b..da328d2b4f 100644 --- a/st2auth/requirements.txt +++ b/st2auth/requirements.txt @@ -6,7 +6,7 @@ # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt bcrypt==3.1.7 -eventlet==0.30.1 +eventlet==0.30.2 git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file git+https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap gunicorn==19.9.0 diff --git a/st2common/requirements.txt b/st2common/requirements.txt index 632f804eec..0d01009b56 100644 --- a/st2common/requirements.txt +++ b/st2common/requirements.txt @@ -10,13 +10,12 @@ apscheduler==3.6.3 chardet<3.1.0 cryptography==3.3.2 dnspython>=1.16.0,<2.0.0 -eventlet==0.30.1 +eventlet==0.30.2 flex==6.14.0 git+https://github.com/StackStorm/orquesta.git@v1.3.0#egg=orquesta git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend gitpython==2.1.15 greenlet==0.4.17 -ipaddr jinja2==2.11.3 jsonpath-rw==1.4.0 jsonschema==2.6.0 diff --git a/st2exporter/requirements.txt b/st2exporter/requirements.txt index 51025d8835..a713318929 100644 --- a/st2exporter/requirements.txt +++ b/st2exporter/requirements.txt @@ -5,7 +5,7 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -eventlet==0.30.1 +eventlet==0.30.2 kombu==4.6.6 oslo.config>=1.12.1,<1.13 six==1.13.0 diff --git a/st2reactor/requirements.txt b/st2reactor/requirements.txt index e5d5696e6a..24047fe257 100644 --- a/st2reactor/requirements.txt +++ b/st2reactor/requirements.txt @@ -6,7 +6,7 @@ # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt apscheduler==3.6.3 -eventlet==0.30.1 +eventlet==0.30.2 jsonpath-rw==1.4.0 jsonschema==2.6.0 kombu==4.6.6 diff --git a/st2stream/requirements.txt b/st2stream/requirements.txt index 7421a46c50..d421659f65 100644 --- a/st2stream/requirements.txt +++ b/st2stream/requirements.txt @@ -5,7 +5,7 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -eventlet==0.30.1 +eventlet==0.30.2 gunicorn==19.9.0 jsonschema==2.6.0 kombu==4.6.6 From fe883e2ddc68cd8ddc918e67389ac8f30dc72f2b Mon Sep 17 00:00:00 2001 From: Carlos Date: Thu, 1 Apr 2021 20:23:50 +0200 Subject: [PATCH 19/58] Run unit tests and integration tests against Python 3.6 and Python 3.8 --- .github/workflows/ci.yaml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c20a0cfd3d..9d432090b4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -44,20 +44,37 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.8] include: - name: 'Lint Checks' task: 'ci-checks' + python-version: "3.6" - name: 'Compile' task: 'ci-compile' + python-version: "3.6" - name: 'Pack Tests' task: 'ci-packs-tests' + python-version: "3.6" - name: 'Unit Tests' task: 'ci-unit' - python-version: '3.6' + python-version: "3.6" - name: 'Integration Tests' task: 'ci-integration' - python-version: '3.6' + python-version: "3.6" + - name: 'Lint Checks' + task: 'ci-checks' + python-version: "3.8" + - name: 'Compile' + task: 'ci-compile' + python-version: "3.8" + - name: 'Pack Tests' + task: 'ci-packs-tests' + python-version: "3.8" + - name: 'Unit Tests' + task: 'ci-unit' + python-version: "3.8" + - name: 'Integration Tests' + task: 'ci-integration' + python-version: "3.8" # This job is slow so we only run in on a daily basis # - name: 'Micro Benchmarks' # task: 'micro-benchmarks' @@ -254,7 +271,7 @@ jobs: shellcheck --version # Print out various environment variables info make play - - name: make + - name: make '${TASK}' # use: script -e -c to print colors run: | script -e -c "make ${TASK}" From b3362e2b2155bade5061ac0f782ad62b0c7fa753 Mon Sep 17 00:00:00 2001 From: Carlos Date: Thu, 1 Apr 2021 22:04:03 +0200 Subject: [PATCH 20/58] Fix linting errors. --- st2common/st2common/persistence/db_init.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/st2common/st2common/persistence/db_init.py b/st2common/st2common/persistence/db_init.py index f43e2fd06b..ed6d080423 100644 --- a/st2common/st2common/persistence/db_init.py +++ b/st2common/st2common/persistence/db_init.py @@ -34,9 +34,7 @@ def _retry_if_connection_error(error): # Ideally, a special execption or atleast some exception code. # If this does become an issue look for "Cannot connect to database" at the # start of error msg. - is_connection_error = isinstance( - error, mongoengine.connection.ConnectionFailure - ) + is_connection_error = isinstance(error, mongoengine.connection.ConnectionFailure) if is_connection_error: LOG.warn("Retry on ConnectionError - %s", error) return is_connection_error From 1c14ae5f7d8a99a2a4401ef2b6a3356a79574d00 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 1 Apr 2021 23:57:15 +0200 Subject: [PATCH 21/58] Handle different response of requests.exceptions.RequestException between Py3.6 and 3.8 --- st2api/tests/unit/controllers/v1/test_packs.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/st2api/tests/unit/controllers/v1/test_packs.py b/st2api/tests/unit/controllers/v1/test_packs.py index 07cacd0be8..d0a3829777 100644 --- a/st2api/tests/unit/controllers/v1/test_packs.py +++ b/st2api/tests/unit/controllers/v1/test_packs.py @@ -17,6 +17,7 @@ import requests import mock +import sys from st2common.content.loader import ContentPackLoader from st2common.models.db.pack import PackDB @@ -345,6 +346,14 @@ def test_index_health(self): def test_index_health_broken(self): resp = self.app.get("/v1/packs/index/health") + + # up to Py3.6 requests.exceptions.RequestExceptuion() appends a trailing , + if sys.version < (3, 8): + broken_index_message = "RequestException('index is broken',)" + else: + broken_index_message = "RequestException('index is broken')" + + self.maxDiff = None self.assertEqual(resp.status_int, 200) self.assertEqual( resp.json, @@ -361,7 +370,7 @@ def test_index_health_broken(self): }, { "url": "http://broken.example.com", - "message": "RequestException('index is broken',)", + "message": broken_index_message, "packs": 0, "error": "unresponsive", }, From 77e2c8520e46daedf3e5ab348e738bcb66412961 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 2 Apr 2021 00:48:09 +0200 Subject: [PATCH 22/58] Fix typo in comment (Excetuion) --- st2api/tests/unit/controllers/v1/test_packs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2api/tests/unit/controllers/v1/test_packs.py b/st2api/tests/unit/controllers/v1/test_packs.py index d0a3829777..8cd2425788 100644 --- a/st2api/tests/unit/controllers/v1/test_packs.py +++ b/st2api/tests/unit/controllers/v1/test_packs.py @@ -347,7 +347,7 @@ def test_index_health_broken(self): resp = self.app.get("/v1/packs/index/health") - # up to Py3.6 requests.exceptions.RequestExceptuion() appends a trailing , + # up to Py3.6 requests.exceptions.RequestException() appends a trailing , if sys.version < (3, 8): broken_index_message = "RequestException('index is broken',)" else: From 4ebe86af8b2f4ada524f80239cbf19c431b343b4 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 2 Apr 2021 00:51:23 +0200 Subject: [PATCH 23/58] Use sys.version_info to retrieve a tuple of the used py release --- st2api/tests/unit/controllers/v1/test_packs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2api/tests/unit/controllers/v1/test_packs.py b/st2api/tests/unit/controllers/v1/test_packs.py index 8cd2425788..7f7e33b079 100644 --- a/st2api/tests/unit/controllers/v1/test_packs.py +++ b/st2api/tests/unit/controllers/v1/test_packs.py @@ -348,7 +348,7 @@ def test_index_health_broken(self): # up to Py3.6 requests.exceptions.RequestException() appends a trailing , - if sys.version < (3, 8): + if sys.version_info < (3, 8): broken_index_message = "RequestException('index is broken',)" else: broken_index_message = "RequestException('index is broken')" From ef6c708e35d053b387c3b84cb21421a2fd936eb1 Mon Sep 17 00:00:00 2001 From: Carlos Date: Fri, 2 Apr 2021 07:41:52 +0200 Subject: [PATCH 24/58] Linting fix. --- st2api/tests/unit/controllers/v1/test_packs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/st2api/tests/unit/controllers/v1/test_packs.py b/st2api/tests/unit/controllers/v1/test_packs.py index 7f7e33b079..eaaf3e3afd 100644 --- a/st2api/tests/unit/controllers/v1/test_packs.py +++ b/st2api/tests/unit/controllers/v1/test_packs.py @@ -346,7 +346,6 @@ def test_index_health(self): def test_index_health_broken(self): resp = self.app.get("/v1/packs/index/health") - # up to Py3.6 requests.exceptions.RequestException() appends a trailing , if sys.version_info < (3, 8): broken_index_message = "RequestException('index is broken',)" From 7a2594958847c7c74f390b1004ba7e45b6ae2881 Mon Sep 17 00:00:00 2001 From: Carlos Date: Fri, 2 Apr 2021 08:36:30 +0200 Subject: [PATCH 25/58] Switch from platform to distro module as the platform module was removed in py3.8 --- contrib/linux/actions/service.py | 2 +- fixed-requirements.txt | 1 + test-requirements.txt | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/linux/actions/service.py b/contrib/linux/actions/service.py index 335e5038f6..4d191fe516 100644 --- a/contrib/linux/actions/service.py +++ b/contrib/linux/actions/service.py @@ -18,7 +18,7 @@ import re import sys import os -import platform +import distro as platform import subprocess from st2common.util.shell import quote_unix diff --git a/fixed-requirements.txt b/fixed-requirements.txt index 7fd4b72cd3..e2e6bf0711 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -10,6 +10,7 @@ cryptography==3.4.7 # Note: 0.20.0 removed select.poll() on which some of our code and libraries we # depend on rely eventlet==0.30.2 +distro==1.5.0 flex==6.14.1 gitpython==2.1.15 # Note: greenlet is used by eventlet diff --git a/test-requirements.txt b/test-requirements.txt index e3c56c26ab..52d5840de7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,6 @@ # NOTE: codecov only supports coverage==4.5.2 coverage==4.5.2 +distro==1.5.0 pep8==1.7.1 st2flake8==0.1.0 astroid==2.5.2 From 72d5ed941f8cbe4c07c60667dcc391e7611bee39 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 2 Apr 2021 13:24:56 +0200 Subject: [PATCH 26/58] Use python3.8 for pre commit hooks. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c539e0b854..aa10e497b3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # files. # This hook relies on development virtual environment being present in virtualenv/. default_language_version: - python: python3.6 + python: python3.8 exclude: '(build|dist)' repos: From d345d6bb6feb6c6ff5dfb7ce271357d0f6020f82 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 3 Apr 2021 17:01:51 +0200 Subject: [PATCH 27/58] Remove tox config - we don't use tox anymore since switching to Python 3. --- Makefile | 1 - tox.ini | 165 ------------------------------------------------------- 2 files changed, 166 deletions(-) delete mode 100644 tox.ini diff --git a/Makefile b/Makefile index dccde6fe66..65510f417a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) SHELL := /bin/bash -TOX_DIR := .tox OS := $(shell uname) # We separate the OSX X and Linux virtualenvs so we can run in a Docker diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 5ec83d0e87..0000000000 --- a/tox.ini +++ /dev/null @@ -1,165 +0,0 @@ -[tox] -envlist = py36-unit,py36-integration.py38-unit,py38-integration -skipsdist = true -skip_missing_interpreters=true - -[testenv] -setenv = PYTHONPATH = {toxinidir}/external - VIRTUALENV_DIR = {envdir} -passenv = NOSE_WITH_TIMER TRAVIS ST2_CI -install_command = pip install -U --force-reinstall {opts} {packages} -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt - -e{toxinidir}/st2tests - -e{toxinidir}/st2actions - -e{toxinidir}/st2api - -e{toxinidir}/st2auth - -e{toxinidir}/st2client - -e{toxinidir}/st2common - -e{toxinidir}/st2reactor - -# Python 3.6 tasks -[testenv:py36-unit] -basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner:{toxinidir}/st2common/tests/runners/mock_query_callback - VIRTUALENV_DIR = {envdir} -passenv = NOSE_WITH_TIMER TRAVIS ST2_CI -install_command = pip install -U --force-reinstall {opts} {packages} -deps = virtualenv - -r{toxinidir}/requirements.txt - -e{toxinidir}/st2client - -e{toxinidir}/st2common - -e{toxinidir}/st2auth -commands = - nosetests --rednose --immediate -sv st2actions/tests/unit/ - nosetests --rednose --immediate -sv st2auth/tests/unit/ - nosetests --rednose --immediate -sv st2api/tests/unit/controllers/v1/ - nosetests --rednose --immediate -sv st2api/tests/unit/controllers/exp/ - nosetests --rednose --immediate -sv st2common/tests/unit/ - nosetests --rednose --immediate -sv st2client/tests/unit/ - nosetests --rednose --immediate -sv st2exporter/tests/unit/ - nosetests --rednose --immediate -sv st2reactor/tests/unit/ - nosetests --rednose --immediate -sv st2stream/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/action_chain_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/inquirer_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/announcement_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/http_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/noop_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/local_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/orquesta_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/python_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/winrm_runner/tests/unit/ - -[testenv:py36-packs] -basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner - VIRTUALENV_DIR = {envdir} -passenv = NOSE_WITH_TIMER TRAVIS ST2_CI -install_command = pip install -U --force-reinstall {opts} {packages} -deps = virtualenv - -r{toxinidir}/requirements.txt - -e{toxinidir}/st2client - -e{toxinidir}/st2common -commands = - st2-run-pack-tests -c -t -x -p contrib/packs - st2-run-pack-tests -c -t -x -p contrib/core - st2-run-pack-tests -c -t -x -p contrib/default - st2-run-pack-tests -c -t -x -p contrib/chatops - st2-run-pack-tests -c -t -x -p contrib/examples - st2-run-pack-tests -c -t -x -p contrib/linux - st2-run-pack-tests -c -t -x -p contrib/hello_st2 - -[testenv:py36-integration] -basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2auth:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner - VIRTUALENV_DIR = {envdir} -passenv = NOSE_WITH_TIMER TRAVIS ST2_CI -install_command = pip install -U --force-reinstall {opts} {packages} -deps = virtualenv - -r{toxinidir}/requirements.txt - -e{toxinidir}/st2client - -e{toxinidir}/st2common -commands = - nosetests --rednose --immediate -sv st2actions/tests/integration/ - nosetests --rednose --immediate -sv st2api/tests/integration/ - nosetests --rednose --immediate -sv st2common/tests/integration/ - nosetests --rednose --immediate -sv st2exporter/tests/integration/ - nosetests --rednose --immediate -sv st2reactor/tests/integration/ - nosetests --rednose --immediate -sv contrib/runners/action_chain_runner/tests/integration/ - nosetests --rednose --immediate -sv contrib/runners/local_runner/tests/integration/ - nosetests --rednose --immediate -sv contrib/runners/orquesta_runner/tests/integration/ - nosetests --rednose --immediate -sv st2tests/integration/orquesta/ - nosetests --rednose --immediate -sv contrib/runners/python_runner/tests/integration/ - -# Python 3.8 tasks -[testenv:py38-unit] -basepython = python3.8 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner:{toxinidir}/st2common/tests/runners/mock_query_callback - VIRTUALENV_DIR = {envdir} -passenv = NOSE_WITH_TIMER TRAVIS ST2_CI -install_command = pip install -U --force-reinstall {opts} {packages} -deps = virtualenv - -r{toxinidir}/requirements.txt - -e{toxinidir}/st2client - -e{toxinidir}/st2common - -e{toxinidir}/st2auth -commands = - nosetests --rednose --immediate -sv st2actions/tests/unit/ - nosetests --rednose --immediate -sv st2auth/tests/unit/ - nosetests --rednose --immediate -sv st2api/tests/unit/controllers/v1/ - nosetests --rednose --immediate -sv st2api/tests/unit/controllers/exp/ - nosetests --rednose --immediate -sv st2common/tests/unit/ - nosetests --rednose --immediate -sv st2client/tests/unit/ - nosetests --rednose --immediate -sv st2exporter/tests/unit/ - nosetests --rednose --immediate -sv st2reactor/tests/unit/ - nosetests --rednose --immediate -sv st2stream/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/action_chain_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/inquirer_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/announcement_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/http_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/noop_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/local_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/orquesta_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/python_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/winrm_runner/tests/unit/ - -[testenv:py38-packs] -basepython = python3.8 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner - VIRTUALENV_DIR = {envdir} -passenv = NOSE_WITH_TIMER TRAVIS ST2_CI -install_command = pip install -U --force-reinstall {opts} {packages} -deps = virtualenv - -r{toxinidir}/requirements.txt - -e{toxinidir}/st2client - -e{toxinidir}/st2common -commands = - st2-run-pack-tests -c -t -x -p contrib/packs - st2-run-pack-tests -c -t -x -p contrib/core - st2-run-pack-tests -c -t -x -p contrib/default - st2-run-pack-tests -c -t -x -p contrib/chatops - st2-run-pack-tests -c -t -x -p contrib/examples - st2-run-pack-tests -c -t -x -p contrib/linux - st2-run-pack-tests -c -t -x -p contrib/hello_st2 - -[testenv:py38-integration] -basepython = python3.8 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2auth:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner - VIRTUALENV_DIR = {envdir} -passenv = NOSE_WITH_TIMER TRAVIS ST2_CI -install_command = pip install -U --force-reinstall {opts} {packages} -deps = virtualenv - -r{toxinidir}/requirements.txt - -e{toxinidir}/st2client - -e{toxinidir}/st2common -commands = - nosetests --rednose --immediate -sv st2actions/tests/integration/ - nosetests --rednose --immediate -sv st2api/tests/integration/ - nosetests --rednose --immediate -sv st2common/tests/integration/ - nosetests --rednose --immediate -sv st2exporter/tests/integration/ - nosetests --rednose --immediate -sv st2reactor/tests/integration/ - nosetests --rednose --immediate -sv contrib/runners/action_chain_runner/tests/integration/ - nosetests --rednose --immediate -sv contrib/runners/local_runner/tests/integration/ - nosetests --rednose --immediate -sv contrib/runners/orquesta_runner/tests/integration/ - nosetests --rednose --immediate -sv st2tests/integration/orquesta/ - nosetests --rednose --immediate -sv contrib/runners/python_runner/tests/integration/ From b6cb22d2f0184b621c123e94dda28a324eefb7ac Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 3 Apr 2021 17:13:57 +0200 Subject: [PATCH 28/58] Update service.py action and ensure it doesn't rely on 3rd party module. This action utilizes remote shell script runner which means we can't rely on any 3rd party library. --- contrib/linux/actions/service.py | 32 ++++++++++++++++++++++++++++++-- fixed-requirements.txt | 1 - test-requirements.txt | 1 - 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/contrib/linux/actions/service.py b/contrib/linux/actions/service.py index 4d191fe516..e370ff6409 100644 --- a/contrib/linux/actions/service.py +++ b/contrib/linux/actions/service.py @@ -15,21 +15,49 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +NOTE: This script file utilizes remote-shell-script runner which means it copied as-is to the +remote host and executed using Python binary available on that systems. + +This means it doesn't use pack or StackStorm specific virtual environment which means we can't +rely on any 3rd party dependencies. +""" + import re import sys import os -import distro as platform +import platform import subprocess from st2common.util.shell import quote_unix -distro = platform.linux_distribution()[0] + +def get_linux_distribution(): + # platform.linux_distribution() is not available in Python >= 3.8 + if hasattr(platform, "linux_distributionx"): + distro = platform.linux_distribution()[0] + else: + # Fall back to shelling out to lsb_release + result = subprocess.run( + "lsb_release -i -s", shell=True, check=True, stdout=subprocess.PIPE + ) + distro = result.stdout.decode("utf-8").strip() + + if not distro: + raise ValueError("Fail to detect distribution we are running on") + + return distro + if len(sys.argv) < 3: raise ValueError("Usage: service.py ") +distro = get_linux_distribution() + args = {"act": quote_unix(sys.argv[1]), "service": quote_unix(sys.argv[2])} +print("Detected distro: %s" % (distro)) + if re.search(distro, "Ubuntu"): if os.path.isfile("/etc/init/%s.conf" % args["service"]): cmd_args = ["service", args["service"], args["act"]] diff --git a/fixed-requirements.txt b/fixed-requirements.txt index e2e6bf0711..7fd4b72cd3 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -10,7 +10,6 @@ cryptography==3.4.7 # Note: 0.20.0 removed select.poll() on which some of our code and libraries we # depend on rely eventlet==0.30.2 -distro==1.5.0 flex==6.14.1 gitpython==2.1.15 # Note: greenlet is used by eventlet diff --git a/test-requirements.txt b/test-requirements.txt index 52d5840de7..e3c56c26ab 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,5 @@ # NOTE: codecov only supports coverage==4.5.2 coverage==4.5.2 -distro==1.5.0 pep8==1.7.1 st2flake8==0.1.0 astroid==2.5.2 From 7b3c053bdab7eec4f6e25b0430a5156b7b9da7c2 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 3 Apr 2021 17:20:15 +0200 Subject: [PATCH 29/58] Make pre-commit hook Python version agnostic. --- .pre-commit-config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa10e497b3..bdd3a3aee6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,9 @@ # files. # This hook relies on development virtual environment being present in virtualenv/. default_language_version: - python: python3.8 + # We don't specify a specific version so it uses either Python 3.6 / Python + # 3.8 depending on what is available locally and on the runner + python: python exclude: '(build|dist)' repos: From 71d5746d1646ca1483bf57f9956b55fdd247f69d Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 3 Apr 2021 17:43:35 +0200 Subject: [PATCH 30/58] Add changelog entry. --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ca8fd664f9..1097348795 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -150,6 +150,10 @@ Changed Contributed by @Kami. +* Update code and dependencies so it supports Python 3.8 and Mongo DB 4.4 #5177 + + Contributed by @nzloshm @winem. + Improvements ~~~~~~~~~~~~ From c2ca91fb007295155a03a9f7f5a5541b9dca17d9 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 3 Apr 2021 17:46:34 +0200 Subject: [PATCH 31/58] Remove testing change. --- contrib/linux/actions/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/linux/actions/service.py b/contrib/linux/actions/service.py index e370ff6409..b022e533d0 100644 --- a/contrib/linux/actions/service.py +++ b/contrib/linux/actions/service.py @@ -34,7 +34,7 @@ def get_linux_distribution(): # platform.linux_distribution() is not available in Python >= 3.8 - if hasattr(platform, "linux_distributionx"): + if hasattr(platform, "linux_distribution"): distro = platform.linux_distribution()[0] else: # Fall back to shelling out to lsb_release From a40322b6d98414e20f1a9e87aa6f44046068ad13 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 3 Apr 2021 17:56:44 +0200 Subject: [PATCH 32/58] Add pylint pragma. --- contrib/linux/actions/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/linux/actions/service.py b/contrib/linux/actions/service.py index b022e533d0..70db65773b 100644 --- a/contrib/linux/actions/service.py +++ b/contrib/linux/actions/service.py @@ -35,7 +35,7 @@ def get_linux_distribution(): # platform.linux_distribution() is not available in Python >= 3.8 if hasattr(platform, "linux_distribution"): - distro = platform.linux_distribution()[0] + distro = platform.linux_distribution()[0] # pylint: disable=no-member else: # Fall back to shelling out to lsb_release result = subprocess.run( From 1a8ab93fa936b0faf06f936ed3a0b6b2fc9b0050 Mon Sep 17 00:00:00 2001 From: Carlos Date: Sat, 3 Apr 2021 19:14:40 +0200 Subject: [PATCH 33/58] Revert disabling monkey patching for threading. --- st2common/st2common/util/monkey_patch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/util/monkey_patch.py b/st2common/st2common/util/monkey_patch.py index ac5b059f39..76b4a191de 100644 --- a/st2common/st2common/util/monkey_patch.py +++ b/st2common/st2common/util/monkey_patch.py @@ -31,7 +31,7 @@ PARENT_ARGS_FLAG = "--parent-args=" -def monkey_patch(patch_thread=False): +def monkey_patch(patch_thread=None): """ Function which performs eventlet monkey patching and also takes into account "--use-debugger" argument in the command line arguments. From 569bc80ab0ccf5fc39c8500aa9008aaed2624a09 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 12:41:53 +0200 Subject: [PATCH 34/58] Add a workaround for tests getting stuck under Python 3.8 due to pymongo code. It turns out the issue is related to us not performing monkey patching early enough in the tests. This is only an issue for tests, for actual service entry point code, the code is already structured correctly so we perform monkey patching as of of the first things we do. --- st2common/st2common/models/db/__init__.py | 21 +++++++++++++++++++++ st2common/st2common/util/service.py | 3 +-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/st2common/st2common/models/db/__init__.py b/st2common/st2common/models/db/__init__.py index ee7261facd..ee839a5bcb 100644 --- a/st2common/st2common/models/db/__init__.py +++ b/st2common/st2common/models/db/__init__.py @@ -15,6 +15,27 @@ from __future__ import absolute_import +import sys + +# NOTE: We need to perform eventlet monkey patching, especially the thread module before importing +# pymongo and mongoengine. If we don't do that, tests will hang because pymongo connection checker +# thread will be constructed before monkey patching. +# This is not an issue for any services where we have code structured correctly so we perform +# monkey patching as early as possible, but this is not always the case with tests and when monkey +# patching happens inside the tests really depends on tests ordering. +# One option would be to simply add monkey_patch() call to the top of every single test file, but +# this would result in tons of duplication. +# Another option is to simply perform monkey patching right here before importing pymongo in case +# we detected we are inside the tests. +# And third option is to re-arrange the imports to we lazily import pymongo when we first need it +# because monkey patching will already be performed by then. +# For now, we go with option 2) since it seems to be good enough of a compromise. +# We detect if we are running inside tests by checking if "nose" module is present - the same logic +# we already use in a couple of other places. +if "nose" in sys.modules.keys(): + from st2common.util.monkey_patch import monkey_patch + monkey_patch() + import copy import importlib import traceback diff --git a/st2common/st2common/util/service.py b/st2common/st2common/util/service.py index e3c2dcb9f9..5824e2e264 100644 --- a/st2common/st2common/util/service.py +++ b/st2common/st2common/util/service.py @@ -15,8 +15,6 @@ from __future__ import absolute_import -import pymongo - from st2common import log as logging @@ -24,6 +22,7 @@ def retry_on_exceptions(exc): + import pymongo LOG.warning("Evaluating retry on exception %s. %s", type(exc), str(exc)) is_mongo_connection_error = isinstance(exc, pymongo.errors.ConnectionFailure) From b79b8eb0e5a2f15fc58495bb76f14f69a349acf2 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 12:47:23 +0200 Subject: [PATCH 35/58] Update CI workflows. --- .github/workflows/ci.yaml | 52 ++++++++----------- .../workflows/orquesta-integration-tests.yaml | 5 ++ 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d8581d99af..aee70b7dd3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -58,6 +58,12 @@ jobs: - name: 'Compile (pip deps, pylint, etc.)' task: 'ci-compile' python-version: '3.6' + - name: 'Lint Checks (black, flake8, etc.)' + task: 'ci-checks' + python-version: '3.8' + - name: 'Compile (pip deps, pylint, etc.)' + task: 'ci-compile' + python-version: '3.8' env: TASK: '${{ matrix.task }}' @@ -150,6 +156,21 @@ jobs: nosetests_node_total: 3 nosetests_node_index: 2 python-version: '3.6' + - name: 'Unit Tests (chunk 1)' + task: 'ci-unit' + nosetests_node_total: 3 + nosetests_node_index: 0 + python-version: '3.3' + - name: 'Unit Tests (chunk 2)' + task: 'ci-unit' + nosetests_node_total: 3 + nosetests_node_index: 1 + python-version: '3.3' + - name: 'Unit Tests (chunk 3)' + task: 'ci-unit' + nosetests_node_total: 3 + nosetests_node_index: 2 + python-version: '3.3' # This job is slow so we only run in on a daily basis # - name: 'Micro Benchmarks' # task: 'micro-benchmarks' @@ -314,36 +335,11 @@ jobs: nosetests_node_total: 3 nosetests_node_index: 2 python-version: '3.6' - - name: 'Lint Checks (black, flake8, etc.)' - task: 'ci-checks' - nosetests_node_total: 1 - nosetests_node_index: 0 - python-version: '3.8' - - name: 'Compile (pip deps, pylint, etc.)' - task: 'ci-compile' - nosetests_node_total: 1 - nosetests_node_index: 0 - python-version: '3.8' - name: 'Pack Tests' task: 'ci-packs-tests' nosetests_node_total: 1 nosetests_node_index: 0 python-version: '3.8' - - name: 'Unit Tests (chunk 1)' - task: 'ci-unit' - nosetests_node_total: 3 - nosetests_node_index: 0 - python-version: '3.8' - - name: 'Unit Tests (chunk 2)' - task: 'ci-unit' - nosetests_node_total: 3 - nosetests_node_index: 1 - python-version: '3.8' - - name: 'Unit Tests (chunk 3)' - task: 'ci-unit' - nosetests_node_total: 3 - nosetests_node_index: 2 - python-version: '3.8' - name: 'Integration Tests (chunk 1)' task: 'ci-integration' nosetests_node_total: 3 @@ -359,12 +355,6 @@ jobs: nosetests_node_total: 3 nosetests_node_index: 2 python-version: '3.8' - # This job is slow so we only run in on a daily basis - # - name: 'Micro Benchmarks' - # task: 'micro-benchmarks' - # python-version: '3.6' - # nosetests_node_total: 1 - # nosetests_node_ index: 0 services: mongo: image: mongo:4.4 diff --git a/.github/workflows/orquesta-integration-tests.yaml b/.github/workflows/orquesta-integration-tests.yaml index 26477b26be..487f695327 100644 --- a/.github/workflows/orquesta-integration-tests.yaml +++ b/.github/workflows/orquesta-integration-tests.yaml @@ -55,6 +55,11 @@ jobs: nosetests_node_total: 1 nosetests_node_index: 0 python-version: '3.6' + - name: 'Integration Tests (Orquesta)' + task: 'ci-orquesta' + nosetests_node_total: 1 + nosetests_node_index: 0 + python-version: '3.8' services: mongo: image: mongo:4.0 From be394a1c9d318c9c5e587650bb80478e11af7c8f Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 12:52:51 +0200 Subject: [PATCH 36/58] Fix lint. --- st2common/st2common/models/db/__init__.py | 1 + st2common/st2common/util/service.py | 1 + 2 files changed, 2 insertions(+) diff --git a/st2common/st2common/models/db/__init__.py b/st2common/st2common/models/db/__init__.py index ee839a5bcb..f3651dc3ee 100644 --- a/st2common/st2common/models/db/__init__.py +++ b/st2common/st2common/models/db/__init__.py @@ -34,6 +34,7 @@ # we already use in a couple of other places. if "nose" in sys.modules.keys(): from st2common.util.monkey_patch import monkey_patch + monkey_patch() import copy diff --git a/st2common/st2common/util/service.py b/st2common/st2common/util/service.py index 5824e2e264..3171eb9983 100644 --- a/st2common/st2common/util/service.py +++ b/st2common/st2common/util/service.py @@ -23,6 +23,7 @@ def retry_on_exceptions(exc): import pymongo + LOG.warning("Evaluating retry on exception %s. %s", type(exc), str(exc)) is_mongo_connection_error = isinstance(exc, pymongo.errors.ConnectionFailure) From 0fa90e8a04a47b6de9eed4b082b750f1ada50648 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 13:41:52 +0200 Subject: [PATCH 37/58] Add missing assignment. --- st2common/st2common/service_setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/st2common/st2common/service_setup.py b/st2common/st2common/service_setup.py index cb3e1433f9..44bfc3d4da 100644 --- a/st2common/st2common/service_setup.py +++ b/st2common/st2common/service_setup.py @@ -136,6 +136,7 @@ def setup( lang_env = os.environ.get("LANG", "unknown") lang_env = os.environ.get("LANG", "notset") pythonioencoding_env = os.environ.get("PYTHONIOENCODING", "notset") + try: language_code, encoding = locale.getdefaultlocale() @@ -145,6 +146,7 @@ def setup( used_locale = "unable to retrieve locale" except Exception as e: used_locale = "unable to retrieve locale: %s " % (str(e)) + encoding = "unknown" LOG.info("Using Python: %s (%s)" % (version, sys.executable)) LOG.info( From 3178d471e3ed96cd65473d1d493d1f46daef7e10 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 14:06:52 +0200 Subject: [PATCH 38/58] Fix some integration tests which fail under Python 3.8 due to the changed behavior. --- st2common/tests/integration/test_logging.py | 12 +++++++++++- .../test_service_setup_log_level_filtering.py | 7 ++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/st2common/tests/integration/test_logging.py b/st2common/tests/integration/test_logging.py index af4eab3bf1..8fea96ef07 100644 --- a/st2common/tests/integration/test_logging.py +++ b/st2common/tests/integration/test_logging.py @@ -18,6 +18,7 @@ import os import sys import signal +import unittest import eventlet from eventlet.green import subprocess @@ -30,7 +31,7 @@ class LogFormattingAndEncodingTestCase(IntegrationTestCase): - def test_formatting_with_unicode_data_works_no_stdout_patching(self): + def test_formatting_with_unicode_data_works_no_stdout_patching_valid_utf8_encoding(self): # Ensure that process doesn't end up in an infinite loop if non-utf8 locale / encoding is # used and a unicode sequence is logged. @@ -66,6 +67,11 @@ def test_formatting_with_unicode_data_works_no_stdout_patching(self): "DEBUG [-] Test debug message with unicode 1 - \u597d\u597d\u597d", stdout ) + @unittest.skipIf(sys.version_info >= (3, 8, 0), "Skipping test under Python >= 3.8") + def test_formatting_with_unicode_data_works_no_stdout_patching_non_valid_utf8_encoding(self): + # Ensure that process doesn't end up in an infinite loop if non-utf8 locale / encoding is + # used and a unicode sequence is logged. + # 2. Process is not using utf-8 encoding - LC_ALL set to invalid locale - should result in # single exception being logged, but not infinite loop process = self._start_process( @@ -105,6 +111,10 @@ def test_formatting_with_unicode_data_works_no_stdout_patching(self): "DEBUG [-] Test debug message with unicode 1 - \u597d\u597d\u597d", stdout ) + def test_formatting_with_unicode_data_works_no_stdout_patching_ascii_pythonioencoding(self): + # Ensure that process doesn't end up in an infinite loop if non-utf8 locale / encoding is + # used and a unicode sequence is logged. + # 3. Process is not using utf-8 encoding - PYTHONIOENCODING set to ascii - should result in # single exception being logged, but not infinite loop process = self._start_process( diff --git a/st2common/tests/integration/test_service_setup_log_level_filtering.py b/st2common/tests/integration/test_service_setup_log_level_filtering.py index 51febaa678..364a016bc7 100644 --- a/st2common/tests/integration/test_service_setup_log_level_filtering.py +++ b/st2common/tests/integration/test_service_setup_log_level_filtering.py @@ -82,6 +82,8 @@ def test_system_info_is_logged_on_startup(self): def test_warning_is_emitted_on_non_utf8_encoding(self): env = os.environ.copy() env["LC_ALL"] = "invalid" + env["ST2_LOG_PATCH_STDOUT"] = "false" + env["PYTHONIOENCODING"] = "ascii" process = self._start_process(config_path=ST2_CONFIG_INFO_LL_PATH, env=env) self.add_process(process=process) @@ -92,7 +94,10 @@ def test_warning_is_emitted_on_non_utf8_encoding(self): # Verify first 4 environment related log messages stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")) self.assertIn("WARNING [-] Detected a non utf-8 locale / encoding", stdout) - self.assertIn("fs encoding: ascii", stdout) + + if sys.version_info < (3, 8, 0): + self.assertIn("fs encoding: ascii", stdout) + self.assertIn("unknown locale: invalid", stdout) def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): From 1afa227793832cd95bb4fb31ecf294a0f674a7cd Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 14:42:45 +0200 Subject: [PATCH 39/58] Fix lint. --- st2common/tests/integration/test_logging.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/st2common/tests/integration/test_logging.py b/st2common/tests/integration/test_logging.py index 8fea96ef07..c7585cb670 100644 --- a/st2common/tests/integration/test_logging.py +++ b/st2common/tests/integration/test_logging.py @@ -31,7 +31,9 @@ class LogFormattingAndEncodingTestCase(IntegrationTestCase): - def test_formatting_with_unicode_data_works_no_stdout_patching_valid_utf8_encoding(self): + def test_formatting_with_unicode_data_works_no_stdout_patching_valid_utf8_encoding( + self, + ): # Ensure that process doesn't end up in an infinite loop if non-utf8 locale / encoding is # used and a unicode sequence is logged. @@ -68,7 +70,9 @@ def test_formatting_with_unicode_data_works_no_stdout_patching_valid_utf8_encodi ) @unittest.skipIf(sys.version_info >= (3, 8, 0), "Skipping test under Python >= 3.8") - def test_formatting_with_unicode_data_works_no_stdout_patching_non_valid_utf8_encoding(self): + def test_formatting_with_unicode_data_works_no_stdout_patching_non_valid_utf8_encoding( + self, + ): # Ensure that process doesn't end up in an infinite loop if non-utf8 locale / encoding is # used and a unicode sequence is logged. @@ -111,7 +115,9 @@ def test_formatting_with_unicode_data_works_no_stdout_patching_non_valid_utf8_en "DEBUG [-] Test debug message with unicode 1 - \u597d\u597d\u597d", stdout ) - def test_formatting_with_unicode_data_works_no_stdout_patching_ascii_pythonioencoding(self): + def test_formatting_with_unicode_data_works_no_stdout_patching_ascii_pythonioencoding( + self, + ): # Ensure that process doesn't end up in an infinite loop if non-utf8 locale / encoding is # used and a unicode sequence is logged. From 9cd9d0f14c210e7a318fa3e6e95ae49e661218ec Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 14:44:17 +0200 Subject: [PATCH 40/58] Try reducing number of jobs we run in parallel to make build matrix more managable. Test and see if the test run time is still in the acceptable time range. --- .github/workflows/ci.yaml | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 90ad18ce5c..17a890fdb4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -143,34 +143,24 @@ jobs: include: - name: 'Unit Tests (chunk 1)' task: 'ci-unit' - nosetests_node_total: 3 + nosetests_node_total: 2 nosetests_node_index: 0 python-version: '3.6' - name: 'Unit Tests (chunk 2)' task: 'ci-unit' - nosetests_node_total: 3 + nosetests_node_total: 2 nosetests_node_index: 1 python-version: '3.6' - - name: 'Unit Tests (chunk 3)' - task: 'ci-unit' - nosetests_node_total: 3 - nosetests_node_index: 2 - python-version: '3.6' - name: 'Unit Tests (chunk 1)' task: 'ci-unit' - nosetests_node_total: 3 + nosetests_node_total: 2 nosetests_node_index: 0 python-version: '3.8' - name: 'Unit Tests (chunk 2)' task: 'ci-unit' - nosetests_node_total: 3 + nosetests_node_total: 2 nosetests_node_index: 1 python-version: '3.8' - - name: 'Unit Tests (chunk 3)' - task: 'ci-unit' - nosetests_node_total: 3 - nosetests_node_index: 2 - python-version: '3.8' # This job is slow so we only run in on a daily basis # - name: 'Micro Benchmarks' # task: 'micro-benchmarks' @@ -322,19 +312,14 @@ jobs: python-version: '3.6' - name: 'Integration Tests (chunk 1)' task: 'ci-integration' - nosetests_node_total: 3 + nosetests_node_total: 2 nosetests_node_index: 0 python-version: '3.6' - name: 'Integration Tests (chunk 2)' task: 'ci-integration' - nosetests_node_total: 3 + nosetests_node_total: 2 nosetests_node_index: 1 python-version: '3.6' - - name: 'Integration Tests (chunk 3)' - task: 'ci-integration' - nosetests_node_total: 3 - nosetests_node_index: 2 - python-version: '3.6' - name: 'Pack Tests' task: 'ci-packs-tests' nosetests_node_total: 1 @@ -342,19 +327,14 @@ jobs: python-version: '3.8' - name: 'Integration Tests (chunk 1)' task: 'ci-integration' - nosetests_node_total: 3 + nosetests_node_total: 2 nosetests_node_index: 0 python-version: '3.8' - name: 'Integration Tests (chunk 2)' task: 'ci-integration' - nosetests_node_total: 3 + nosetests_node_total: 2 nosetests_node_index: 1 python-version: '3.8' - - name: 'Integration Tests (chunk 3)' - task: 'ci-integration' - nosetests_node_total: 3 - nosetests_node_index: 2 - python-version: '3.8' services: mongo: image: mongo:4.4 From 7b04da737e7d3ad213ac2bf9f53e0b7161fa68f4 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 14:49:09 +0200 Subject: [PATCH 41/58] Fix some bad merge conflict resolution. --- .github/workflows/ci.yaml | 52 +++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 17a890fdb4..2792f57985 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -name: ci +name: CI on: push: @@ -20,9 +20,6 @@ on: # run every night at midnight - cron: '0 0 * * *' -# TODO: Our workflow is far from ideal. We need to refactor it into multiple -# ones and only run commands which are needed for some steps for those steps and -# not for all jobs: # Special job which automatically cancels old runs for the same branch, prevents runs for the # same file set which has already passed, etc. @@ -119,7 +116,7 @@ jobs: ./scripts/ci/print-versions.sh - name: make # use: script -e -c to print colors - timeout-minutes: 25 + timeout-minutes: 7 run: | script -e -c "make ${TASK}" - name: Nightly @@ -276,7 +273,10 @@ jobs: ./scripts/ci/print-versions.sh - name: make # use: script -e -c to print colors - timeout-minutes: 25 + # TODO: Use dynamic timeout value based on the branch - for master we + # need to use timeout x2 due to coverage overhead + # timeout-minutes: 8 + timeout-minutes: 14 run: | script -e -c "make ${TASK}" - name: Nightly @@ -486,36 +486,30 @@ jobs: ./scripts/ci/print-versions.sh - name: make - timeout-minutes: 25 # may die if rabbitmq fails to start - env: - MAX_ATTEMPTS: 3 - RETRY_DELAY: 5 + if: "${{ env.TASK == 'ci-integration' }}" + #timeout-minutes: 7 + # TODO: Use dynamic timeout value based on the branch - for master we + # need to use timeout x2 due to coverage overhead + timeout-minutes: 14 # may die if rabbitmq fails to start # use: script -e -c to print colors run: | - # There is a race in some orequesta integration tests so they tend to fail quite often. - # To avoid needed to re-run whole workflow in such case, we should try to retry this - # specific step. This saves us a bunch of time manually re-running the whole workflow. - # TODO: Try to identify problematic tests (iirc mostly orquesta ones) and only retry / - # re-run those. - set +e - for i in $(seq 1 ${MAX_ATTEMPTS}); do - echo "Attempt: ${i}/${MAX_ATTEMPTS}" - - script -e -c "make ${TASK}" && exit 0 - - echo "Command failed, will retry in ${RETRY_DELAY} seconds..." - sleep ${RETRY_DELAY} - done - - set -e - echo "Failed after ${MAX_ATTEMPTS} attempts, failing the job." - exit 1 + script -e -c "make ${TASK}" && exit 0 - name: Codecov # NOTE: We only generate and submit coverage report for master and version branches and only when the build succeeds (default on GitHub Actions, this was not the case on Travis so we had to explicitly check success) if: "${{ success() && env.ENABLE_COVERAGE == 'yes' && env.TASK == 'ci-integration' }}" - timeout-minutes: 25 run: | ./scripts/ci/submit-codecov-coverage.sh + - name: Compress Service Logs Before upload + if: ${{ failure() && env.TASK == 'ci-integration' }} + run: | + tar cvzpf logs.tar.gz logs/* + - name: Upload StackStorm services Logs + if: ${{ failure() && env.TASK == 'ci-integration' }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: logs.tar.gz + retention-days: 7 - name: Stop Redis Service Container if: "${{ always() }}" run: docker rm --force redis || true From 26a27553e634d403639a6cf1f3976497ed7c4b97 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 14:50:06 +0200 Subject: [PATCH 42/58] Also use MongoDB 4.4 for orquesta integration tests. --- .github/workflows/orquesta-integration-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/orquesta-integration-tests.yaml b/.github/workflows/orquesta-integration-tests.yaml index 487f695327..7f520b41e2 100644 --- a/.github/workflows/orquesta-integration-tests.yaml +++ b/.github/workflows/orquesta-integration-tests.yaml @@ -62,7 +62,7 @@ jobs: python-version: '3.8' services: mongo: - image: mongo:4.0 + image: mongo:4.4 ports: - 27017:27017 From d36559a9b188cb20fd31d52bb9432ec04b9bf7d3 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 14:56:11 +0200 Subject: [PATCH 43/58] For consistency, split it into multiple test methods. --- st2common/tests/integration/test_logging.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/st2common/tests/integration/test_logging.py b/st2common/tests/integration/test_logging.py index c7585cb670..2e9c16c7e3 100644 --- a/st2common/tests/integration/test_logging.py +++ b/st2common/tests/integration/test_logging.py @@ -160,7 +160,9 @@ def test_formatting_with_unicode_data_works_no_stdout_patching_ascii_pythonioenc "DEBUG [-] Test debug message with unicode 1 - \u597d\u597d\u597d", stdout ) - def test_formatting_with_unicode_data_works_with_stdout_patching(self): + def test_formatting_with_unicode_data_works_with_stdout_patching_valid_utf8_encoding( + self, + ): # Test a scenario where patching is enabled which means it should never result in infinite # loop # 1. Process is using a utf-8 encoding @@ -180,7 +182,6 @@ def test_formatting_with_unicode_data_works_with_stdout_patching(self): stdout = process.stdout.read().decode("utf-8") stderr = process.stderr.read().decode("utf-8") stdout_lines = stdout.split("\n") - print(stderr) self.assertEqual(stderr, "") self.assertTrue(len(stdout_lines) < 20) @@ -196,6 +197,9 @@ def test_formatting_with_unicode_data_works_with_stdout_patching(self): "DEBUG [-] Test debug message with unicode 1 - \u597d\u597d\u597d", stdout ) + def test_formatting_with_unicode_data_works_with_stdout_patching_non_valid_utf8_encoding( + self, + ): # 2. Process is not using utf-8 encoding process = self._start_process( env={ @@ -212,6 +216,7 @@ def test_formatting_with_unicode_data_works_with_stdout_patching(self): stdout = process.stdout.read().decode("utf-8") stderr = process.stderr.read().decode("utf-8") + stdout_lines = stdout.split("\n") self.assertEqual(stderr, "") self.assertTrue(len(stdout_lines) < 50) @@ -227,6 +232,9 @@ def test_formatting_with_unicode_data_works_with_stdout_patching(self): "DEBUG [-] Test debug message with unicode 1 - \u597d\u597d\u597d", stdout ) + def test_formatting_with_unicode_data_works_with_stdout_patching__ascii_pythonioencoding( + self, + ): # 3. Process is not using utf-8 encoding - PYTHONIOENCODING set to ascii process = self._start_process( env={ From 8f2597786aae51bb326535b61d97c26a9787d581 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 15:00:12 +0200 Subject: [PATCH 44/58] Update comment. --- st2common/st2common/models/db/__init__.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/st2common/st2common/models/db/__init__.py b/st2common/st2common/models/db/__init__.py index f3651dc3ee..cd87eed33b 100644 --- a/st2common/st2common/models/db/__init__.py +++ b/st2common/st2common/models/db/__init__.py @@ -19,19 +19,23 @@ # NOTE: We need to perform eventlet monkey patching, especially the thread module before importing # pymongo and mongoengine. If we don't do that, tests will hang because pymongo connection checker -# thread will be constructed before monkey patching. -# This is not an issue for any services where we have code structured correctly so we perform +# thread will be constructed before monkey patching with reference to non patched threading module. +# This is not an issue for any service where we have code structured correctly so we perform # monkey patching as early as possible, but this is not always the case with tests and when monkey -# patching happens inside the tests really depends on tests ordering. +# patching happens inside the tests really depends on tests import ordering, etc. +# # One option would be to simply add monkey_patch() call to the top of every single test file, but # this would result in tons of duplication. +# # Another option is to simply perform monkey patching right here before importing pymongo in case -# we detected we are inside the tests. -# And third option is to re-arrange the imports to we lazily import pymongo when we first need it -# because monkey patching will already be performed by then. -# For now, we go with option 2) since it seems to be good enough of a compromise. -# We detect if we are running inside tests by checking if "nose" module is present - the same logic -# we already use in a couple of other places. +# we detected we are running inside tests. +# +# And third option is to re-arrange the imports so we lazily import pymongo + mongoengine when we +# first need it because monkey patching will already be performed by then. +# +# For now, we go with option 2) since it seems to be good enough of a compromise. We detect if we +# are running inside tests by checking if "nose" module is present - the same logic we already use +# in a couple of other places (and something which would need to be changed if we switch to pytest). if "nose" in sys.modules.keys(): from st2common.util.monkey_patch import monkey_patch From f07ace0dc9335f04eedc670f68bc120c652a3b3f Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 15:03:14 +0200 Subject: [PATCH 45/58] Add a workaround for a failing test. --- .../tests/integration/test_pythonrunner_behavior.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py index a694ffceac..38441933b2 100644 --- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py +++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py @@ -21,6 +21,10 @@ import mock import tempfile +from st2common.util.monkey_patch import use_select_poll_workaround + +use_select_poll_workaround() + from oslo_config import cfg from python_runner import python_runner From 4827e6154f275eac3a8887df5167144db8821214 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 15:10:16 +0200 Subject: [PATCH 46/58] Print stdout to make troubleshooting easier. --- st2common/tests/integration/test_logging.py | 1 + .../tests/integration/test_service_setup_log_level_filtering.py | 1 + 2 files changed, 2 insertions(+) diff --git a/st2common/tests/integration/test_logging.py b/st2common/tests/integration/test_logging.py index 2e9c16c7e3..b84a248e7c 100644 --- a/st2common/tests/integration/test_logging.py +++ b/st2common/tests/integration/test_logging.py @@ -219,6 +219,7 @@ def test_formatting_with_unicode_data_works_with_stdout_patching_non_valid_utf8_ stdout_lines = stdout.split("\n") self.assertEqual(stderr, "") + print(stdout) self.assertTrue(len(stdout_lines) < 50) self.assertIn("INFO [-] Test info message 1", stdout) diff --git a/st2common/tests/integration/test_service_setup_log_level_filtering.py b/st2common/tests/integration/test_service_setup_log_level_filtering.py index 364a016bc7..99175c943a 100644 --- a/st2common/tests/integration/test_service_setup_log_level_filtering.py +++ b/st2common/tests/integration/test_service_setup_log_level_filtering.py @@ -74,6 +74,7 @@ def test_system_info_is_logged_on_startup(self): # Verify first 4 environment related log messages stdout = process.stdout.read().decode("utf-8") + print(stdout) self.assertIn("INFO [-] Using Python:", stdout) self.assertIn("INFO [-] Using fs encoding:", stdout) self.assertIn("INFO [-] Using config files:", stdout) From a6d3faa2dce7c6be898405857c381b690656ebb3 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 16:36:03 +0200 Subject: [PATCH 47/58] Fix test so it passes on CI, update base test class to make sure we kill any stray processes on setUp() in addition to tearDown(). --- st2common/tests/integration/test_logging.py | 2 +- st2tests/st2tests/base.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/st2common/tests/integration/test_logging.py b/st2common/tests/integration/test_logging.py index b84a248e7c..997f96c7a5 100644 --- a/st2common/tests/integration/test_logging.py +++ b/st2common/tests/integration/test_logging.py @@ -220,7 +220,7 @@ def test_formatting_with_unicode_data_works_with_stdout_patching_non_valid_utf8_ self.assertEqual(stderr, "") print(stdout) - self.assertTrue(len(stdout_lines) < 50) + self.assertTrue(len(stdout_lines) < 100) self.assertIn("INFO [-] Test info message 1", stdout) self.assertIn("Test debug message 1", stdout) diff --git a/st2tests/st2tests/base.py b/st2tests/st2tests/base.py index de90571684..f1237978b4 100644 --- a/st2tests/st2tests/base.py +++ b/st2tests/st2tests/base.py @@ -551,12 +551,20 @@ class IntegrationTestCase(TestCase): processes = {} + def setUp(self): + super(IntegrationTestCase, self).setUp() + self._stop_running_processes() + def tearDown(self): super(IntegrationTestCase, self).tearDown() + self._stop_running_processes() + def _stop_running_processes(self): # Make sure we kill all the processes on teardown so they don't linger around if an # exception was thrown. - for pid, process in self.processes.items(): + pids = list(self.processes.keys()) + for pid in pids: + process = self.processes[pid] try: process.kill() @@ -564,6 +572,8 @@ def tearDown(self): # Process already exited or similar pass + del self.processes[pid] + if self.print_stdout_stderr_on_teardown: try: stdout = process.stdout.read() From 21abd0b4f136f764cec1831ae77cb9085acf76f2 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 16:53:21 +0200 Subject: [PATCH 48/58] Add workaround for tests, add additional logging. --- st2common/st2common/util/pack_management.py | 5 +++++ .../integration/test_service_setup_log_level_filtering.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/st2common/st2common/util/pack_management.py b/st2common/st2common/util/pack_management.py index 0fde5b1d86..12d0251e00 100644 --- a/st2common/st2common/util/pack_management.py +++ b/st2common/st2common/util/pack_management.py @@ -26,6 +26,11 @@ import stat import re +# This test workaround needs to be used before importing git +from st2common.util.monkey_patch import use_select_poll_workaround + +use_select_poll_workaround() + import six from git.repo import Repo from gitdb.exc import BadName, BadObject diff --git a/st2common/tests/integration/test_service_setup_log_level_filtering.py b/st2common/tests/integration/test_service_setup_log_level_filtering.py index 99175c943a..8419e95b68 100644 --- a/st2common/tests/integration/test_service_setup_log_level_filtering.py +++ b/st2common/tests/integration/test_service_setup_log_level_filtering.py @@ -74,7 +74,11 @@ def test_system_info_is_logged_on_startup(self): # Verify first 4 environment related log messages stdout = process.stdout.read().decode("utf-8") + print("=====") print(stdout) + print("=====") + print(stdout) + print("=====") self.assertIn("INFO [-] Using Python:", stdout) self.assertIn("INFO [-] Using fs encoding:", stdout) self.assertIn("INFO [-] Using config files:", stdout) From e22734d55cdd7e0fbf54f809d24ae8fa0c35feb6 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 17:00:30 +0200 Subject: [PATCH 49/58] Remove debug logging. --- .../integration/test_service_setup_log_level_filtering.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/st2common/tests/integration/test_service_setup_log_level_filtering.py b/st2common/tests/integration/test_service_setup_log_level_filtering.py index 8419e95b68..364a016bc7 100644 --- a/st2common/tests/integration/test_service_setup_log_level_filtering.py +++ b/st2common/tests/integration/test_service_setup_log_level_filtering.py @@ -74,11 +74,6 @@ def test_system_info_is_logged_on_startup(self): # Verify first 4 environment related log messages stdout = process.stdout.read().decode("utf-8") - print("=====") - print(stdout) - print("=====") - print(stdout) - print("=====") self.assertIn("INFO [-] Using Python:", stdout) self.assertIn("INFO [-] Using fs encoding:", stdout) self.assertIn("INFO [-] Using config files:", stdout) From 8c8346414a00463c559fb97b888a8eb8926403db Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 18:04:08 +0200 Subject: [PATCH 50/58] Also utilize mongo 4.4 for unit tests. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2792f57985..c88baf9207 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -166,7 +166,7 @@ jobs: # nosetests_node_ index: 0 services: mongo: - image: mongo:4.0 + image: mongo:4.4 ports: - 27017:27017 From 6684ff59d1a5581c8e273d0d894f696e66f1af80 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 18:16:46 +0200 Subject: [PATCH 51/58] Looks like cache is corrupted, try using a new one. --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c88baf9207..a90bcb4e45 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -86,9 +86,9 @@ jobs: # TODO: maybe make the virtualenv a partial cache to exclude st2*? # !virtualenv/lib/python*/site-packages/st2* # !virtualenv/bin/st2* - key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} + key: ${{ runner.os }}-v2-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} restore-keys: | - ${{ runner.os }}-python-${{ matrix.python }}- + ${{ runner.os }}-v2-python-${{ matrix.python }}- - name: Cache APT Dependencies id: cache-apt-deps uses: actions/cache@v2 From be8113a5cfde3f55ead4232e2756b38d47a529de Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 18:29:48 +0200 Subject: [PATCH 52/58] Use longer sleep to avoid occasional race. --- .../integration/test_service_setup_log_level_filtering.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/st2common/tests/integration/test_service_setup_log_level_filtering.py b/st2common/tests/integration/test_service_setup_log_level_filtering.py index 364a016bc7..0907dcbb5d 100644 --- a/st2common/tests/integration/test_service_setup_log_level_filtering.py +++ b/st2common/tests/integration/test_service_setup_log_level_filtering.py @@ -69,7 +69,7 @@ def test_system_info_is_logged_on_startup(self): self.add_process(process=process) # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(4) process.send_signal(signal.SIGKILL) # Verify first 4 environment related log messages @@ -88,7 +88,7 @@ def test_warning_is_emitted_on_non_utf8_encoding(self): self.add_process(process=process) # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(4) process.send_signal(signal.SIGKILL) # Verify first 4 environment related log messages @@ -106,7 +106,7 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): self.add_process(process=process) # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(4) process.send_signal(signal.SIGKILL) # Verify first 4 environment related log messages @@ -123,7 +123,7 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): self.add_process(process=process) # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(4) process.send_signal(signal.SIGKILL) # First 6 log lines are debug messages about the environment which are always logged From 30daa7fcd8f07f4e4a4ef64ca3eee3d14c8bbbf8 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 18:36:19 +0200 Subject: [PATCH 53/58] Pin all st2client deps to specific versions. --- fixed-requirements.txt | 2 ++ requirements.txt | 4 ++-- st2client/requirements.txt | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/fixed-requirements.txt b/fixed-requirements.txt index f10fbd36b4..011184509b 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -42,6 +42,8 @@ retrying==1.3.3 routes==2.4.1 semver==2.9.0 six==1.13.0 +argparse==1.12.2 +prettytable==2.1.0 # NOTE: sseclient has various issues which sometimes hang the connection for a long time, etc. sseclient-py==1.7 stevedore==1.30.1 diff --git a/requirements.txt b/requirements.txt index 1f8a7d3841..89548084dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ RandomWords amqp==2.5.2 apscheduler==3.7.0 -argcomplete +argcomplete==1.12.2 bcrypt==3.2.0 chardet<3.1.0 cryptography==3.4.7 @@ -39,7 +39,7 @@ oslo.config>=1.12.1,<1.13 oslo.utils<5.0,>=4.0.0 paramiko==2.7.2 passlib==1.7.4 -prettytable +prettytable==2.1.0 prompt-toolkit==1.0.15 psutil==5.8.0 pyinotify==0.9.6; platform_system=="Linux" diff --git a/st2client/requirements.txt b/st2client/requirements.txt index 5103232989..51992bc8cb 100644 --- a/st2client/requirements.txt +++ b/st2client/requirements.txt @@ -5,13 +5,13 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -argcomplete +argcomplete==1.12.2 chardet<3.1.0 cryptography==3.4.7 jsonpath-rw==1.4.0 jsonschema==2.6.0 orjson==3.5.1 -prettytable +prettytable==2.1.0 prompt-toolkit==1.0.15 python-dateutil==2.8.1 python-editor==1.0.4 From f3f2d6b834fef9005c91cd43be727b332c721e50 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 19:10:51 +0200 Subject: [PATCH 54/58] Eplicitly pin importlib-metadata. Recently st2client tests started failing due to new version of importlib-metadata being published which is not supported by one of our transitive dependencies. --- requirements.txt | 1 + st2client/in-requirements.txt | 1 + st2client/requirements.txt | 1 + 3 files changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 89548084dd..deccc4bf23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,6 +23,7 @@ git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backe gitpython==2.1.15 greenlet==1.0.0 gunicorn==20.1.0 +importlib-metadata==3.10.1 jinja2==2.11.3 jsonpath-rw==1.4.0 jsonschema==2.6.0 diff --git a/st2client/in-requirements.txt b/st2client/in-requirements.txt index 6d1341e742..da020c46df 100644 --- a/st2client/in-requirements.txt +++ b/st2client/in-requirements.txt @@ -1,4 +1,5 @@ # Remember to list implicit packages here, otherwise version won't be fixated! +importlib-metadata argcomplete prettytable pytz diff --git a/st2client/requirements.txt b/st2client/requirements.txt index 51992bc8cb..f8d6c9ba98 100644 --- a/st2client/requirements.txt +++ b/st2client/requirements.txt @@ -8,6 +8,7 @@ argcomplete==1.12.2 chardet<3.1.0 cryptography==3.4.7 +importlib-metadata==3.10.1 jsonpath-rw==1.4.0 jsonschema==2.6.0 orjson==3.5.1 From 61d5520eb124ecaf55aa18203827f3a51ac2d9f8 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 19:28:41 +0200 Subject: [PATCH 55/58] Add support for new database.compressors config option with which user can specify which compressor client should try to use when talking to MongoDB (aka for transport / network level compression). Default (same as before) is no transport level compression. --- CHANGELOG.rst | 16 +++++ conf/st2.conf.sample | 4 ++ fixed-requirements.txt | 3 + requirements.txt | 1 + st2common/in-requirements.txt | 2 + st2common/requirements.txt | 1 + st2common/st2common/config.py | 14 +++++ st2common/st2common/models/db/__init__.py | 11 ++++ st2common/tests/unit/test_db.py | 74 +++++++++++++++++++++++ 9 files changed, 126 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e0eb71a0e7..a79a2b042a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -49,6 +49,22 @@ Added Contributed by @cognifloyd. +* Add new ``database.compressors`` and ``database.zlib_compression_level`` config option which + specifies compression algorithms client supports for network / transport level compression + when talking to MongoDB. + + Actual compression algorithm used will be then decided by the server and depends on the + algorithms which are supported by the server + client. + + Possible / valid values include: zstd, zlib. Keep in mind that zstandard (zstd) is only supported + by MongoDB >= 4.2. + + Our official Debian and RPM packages bundle ``zstandard`` dependency by default which means + setting this value to ``zstd`` should work out of the box as long as the server runs + MongoDB >= 4.2. #5177 + + Contributed by @Kami. + Changed ~~~~~~~ diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index 488939eb55..97885cc5a7 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -113,6 +113,8 @@ url = None [database] # Specifies database authentication mechanisms. By default, it use SCRAM-SHA-1 with MongoDB 3.0 and later, MONGODB-CR (MongoDB Challenge Response protocol) for older servers. authentication_mechanism = None +# Comma delimited string of compression algorithms to use for transport level compression. Actual algorithm will then be decided based on the algorithms supported by the client and the server. For example: zstd. Defaults to no compression. Keep in mind that zstd is only supported with MongoDB 4.2 and later. +compressors = # Connection retry backoff max (seconds). connection_retry_backoff_max_s = 10 # Backoff multiplier (seconds). @@ -143,6 +145,8 @@ ssl_keyfile = None ssl_match_hostname = True # username for db login username = None +# Compression level when compressors is set to zlib. Valid calues are -1 to 9. Defaults to 6. +zlib_compression_level = [exporter] # Directory to dump data to. diff --git a/fixed-requirements.txt b/fixed-requirements.txt index 011184509b..b11c2398d7 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -31,6 +31,7 @@ prance==0.9.0 prompt-toolkit==1.0.15 pyinotify==0.9.6; platform_system=="Linux" pymongo==3.11.3 +zstandard==0.15.2 python-editor==1.0.4 python-keyczar==0.716 pytz==2021.1 @@ -43,7 +44,9 @@ routes==2.4.1 semver==2.9.0 six==1.13.0 argparse==1.12.2 +argcomplete==1.12.2 prettytable==2.1.0 +importlib-metadata==3.10.1 # NOTE: sseclient has various issues which sometimes hang the connection for a long time, etc. sseclient-py==1.7 stevedore==1.30.1 diff --git a/requirements.txt b/requirements.txt index deccc4bf23..ec14b5b53e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -69,3 +69,4 @@ unittest2 webob==1.8.7 webtest zake==0.2.2 +zstandard==0.15.2 diff --git a/st2common/in-requirements.txt b/st2common/in-requirements.txt index c427e34614..9307d21057 100644 --- a/st2common/in-requirements.txt +++ b/st2common/in-requirements.txt @@ -16,6 +16,8 @@ oslo.config paramiko pyyaml pymongo +# used for optional network level compression for mongodb +zstandard cryptography requests retrying diff --git a/st2common/requirements.txt b/st2common/requirements.txt index 78ae5b9a9e..8e78c26d1e 100644 --- a/st2common/requirements.txt +++ b/st2common/requirements.txt @@ -40,3 +40,4 @@ tooz==2.8.0 udatetime==0.0.16 webob==1.8.7 zake==0.2.2 +zstandard==0.15.2 diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index 0429c81ea2..9d39f86d64 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -243,6 +243,20 @@ def register_opts(ignore_errors=False): "By default, it use SCRAM-SHA-1 with MongoDB 3.0 and later, " "MONGODB-CR (MongoDB Challenge Response protocol) for older servers.", ), + cfg.StrOpt( + "compressors", + default="", + help="Comma delimited string of compression algorithms to use for transport level " + "compression. Actual algorithm will then be decided based on the algorithms " + "supported by the client and the server. For example: zstd. Defaults to no " + "compression. Keep in mind that zstd is only supported with MongoDB 4.2 and later.", + ), + cfg.IntOpt( + "zlib_compression_level", + default="", + help="Compression level when compressors is set to zlib. Valid calues are -1 to 9. " + "Defaults to 6.", + ), ] do_register_opts(db_opts, "database", ignore_errors) diff --git a/st2common/st2common/models/db/__init__.py b/st2common/st2common/models/db/__init__.py index cd87eed33b..9ed99c7c91 100644 --- a/st2common/st2common/models/db/__init__.py +++ b/st2common/st2common/models/db/__init__.py @@ -167,6 +167,16 @@ def _db_connect( ssl_match_hostname=ssl_match_hostname, ) + compressor_kwargs = {} + + if cfg.CONF.database.compressors: + compressor_kwargs["compressors"] = cfg.CONF.database.compressors + + if cfg.CONF.database.zlib_compression_level is not None: + compressor_kwargs[ + "zlibCompressionLevel" + ] = cfg.CONF.database.zlib_compression_level + # NOTE: We intentionally set "serverSelectionTimeoutMS" to 3 seconds. By default it's set to # 30 seconds, which means it will block up to 30 seconds and fail if there are any SSL related # or other errors @@ -181,6 +191,7 @@ def _db_connect( connectTimeoutMS=connection_timeout, serverSelectionTimeoutMS=connection_timeout, **ssl_kwargs, + **compressor_kwargs, ) # NOTE: Since pymongo 3.0, connect() method is lazy and not blocking (always returns success) diff --git a/st2common/tests/unit/test_db.py b/st2common/tests/unit/test_db.py index da0157127e..6268bb3dc9 100644 --- a/st2common/tests/unit/test_db.py +++ b/st2common/tests/unit/test_db.py @@ -103,10 +103,12 @@ class DbConnectionTestCase(DbTestCase): def setUp(self): # NOTE: It's important we re-establish a connection on each setUp self.setUpClass() + cfg.CONF.reset() def tearDown(self): # NOTE: It's important we disconnect here otherwise tests will fail disconnect() + cfg.CONF.reset() def test_check_connect(self): """ @@ -121,6 +123,78 @@ def test_check_connect(self): ) self.assertIn(expected_str, str(client), "Not connected to desired host.") + def test_network_level_compression(self): + disconnect() + + db_name = "st2" + db_host = "localhost" + db_port = 27017 + + # If running version < MongoDB 4.2 we skip this check since zstd is only supported in server + # >= 4.2 + connection = db_setup(db_name=db_name, db_host=db_host, db_port=db_port) + server_version = tuple( + [int(x) for x in connection.server_info()["version"].split(".")] + ) + + if server_version < (4, 2, 0): + self.skipTest("Skipping test since running MongoDB < 4.2") + return + + disconnect() + + # 1. Verify default is no compression + connection = db_setup(db_name=db_name, db_host=db_host, db_port=db_port) + # Sadly there is no nicer way to assert that it seems + self.assertFalse("compressors=['zstd']" in str(connection)) + self.assertFalse("compressors" in str(connection)) + + # 2. Verify using zstd works - specified using config option + disconnect() + + cfg.CONF.set_override(name="compressors", group="database", override="zstd") + + connection = db_setup(db_name=db_name, db_host=db_host, db_port=db_port) + # Sadly there is no nicer way to assert that it seems + self.assertTrue("compressors=['zstd']" in str(connection)) + + # 3. Verify using zstd works - specified inside URI + disconnect() + + cfg.CONF.set_override(name="compressors", group="database", override=None) + db_host = "mongodb://127.0.0.1/?compressors=zstd" + + connection = db_setup(db_name=db_name, db_host=db_host, db_port=db_port) + # Sadly there is no nicer way to assert that it seems + self.assertTrue("compressors=['zstd']" in str(connection)) + + # 4. Verify using zlib works - specified using config option + disconnect() + + cfg.CONF.set_override(name="compressors", group="database", override="zlib") + cfg.CONF.set_override( + name="zlib_compression_level", group="database", override=8 + ) + + connection = db_setup(db_name=db_name, db_host=db_host, db_port=db_port) + # Sadly there is no nicer way to assert that it seems + self.assertTrue("compressors=['zlib']" in str(connection)) + self.assertTrue("zlibcompressionlevel=8" in str(connection)) + + # 5. Verify using zlib works - specified inside URI + disconnect() + + cfg.CONF.set_override(name="compressors", group="database", override=None) + cfg.CONF.set_override( + name="zlib_compression_level", group="database", override=None + ) + db_host = "mongodb://127.0.0.1/?compressors=zlib&zlibCompressionLevel=9" + + connection = db_setup(db_name=db_name, db_host=db_host, db_port=db_port) + # Sadly there is no nicer way to assert that it seems + self.assertTrue("compressors=['zlib']" in str(connection)) + self.assertTrue("zlibcompressionlevel=9" in str(connection)) + def test_get_ssl_kwargs(self): # 1. No SSL kwargs provided ssl_kwargs = _get_ssl_kwargs() From e776a29f9ba2b9d7eb0003438376a6c3e9bbd526 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 19:55:18 +0200 Subject: [PATCH 56/58] Add print statements for easier troubleshooting on failure. --- .../integration/test_service_setup_log_level_filtering.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/st2common/tests/integration/test_service_setup_log_level_filtering.py b/st2common/tests/integration/test_service_setup_log_level_filtering.py index 0907dcbb5d..ba1e172234 100644 --- a/st2common/tests/integration/test_service_setup_log_level_filtering.py +++ b/st2common/tests/integration/test_service_setup_log_level_filtering.py @@ -74,6 +74,9 @@ def test_system_info_is_logged_on_startup(self): # Verify first 4 environment related log messages stdout = process.stdout.read().decode("utf-8") + stderr = process.stderr.read().decode("utf-8") + print(stdout) + print(stderr) self.assertIn("INFO [-] Using Python:", stdout) self.assertIn("INFO [-] Using fs encoding:", stdout) self.assertIn("INFO [-] Using config files:", stdout) @@ -93,6 +96,9 @@ def test_warning_is_emitted_on_non_utf8_encoding(self): # Verify first 4 environment related log messages stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")) + stderr = process.stderr.read().decode("utf-8") + print(stdout) + print(stderr) self.assertIn("WARNING [-] Detected a non utf-8 locale / encoding", stdout) if sys.version_info < (3, 8, 0): From 09e0dedb84ff82cc46d9221eca9594d0008decf6 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 19:59:20 +0200 Subject: [PATCH 57/58] Update test class to clean environment pre and post test method run to avoid leaking env variables into other tests. --- .../test_service_setup_log_level_filtering.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/st2common/tests/integration/test_service_setup_log_level_filtering.py b/st2common/tests/integration/test_service_setup_log_level_filtering.py index ba1e172234..6b91a0874a 100644 --- a/st2common/tests/integration/test_service_setup_log_level_filtering.py +++ b/st2common/tests/integration/test_service_setup_log_level_filtering.py @@ -63,6 +63,21 @@ class ServiceSetupLogLevelFilteringTestCase(IntegrationTestCase): + def setUp(self): + super(ServiceSetupLogLevelFilteringTestCase, self).setUp() + self._reset_env() + + def tearDown(self): + super(ServiceSetupLogLevelFilteringTestCase, self).tearDown() + self._reset_env() + + def _reset_env(self): + keys_to_delete = ["LC_ALL", "ST2_LOG_PATCH_STDOUT", "PYTHONIOENCODING"] + + for key in keys_to_delete: + if key in os.environ: + del os.environ[key] + def test_system_info_is_logged_on_startup(self): # Verify INFO level service start up messages process = self._start_process(config_path=ST2_CONFIG_INFO_LL_PATH) From 91cadcd1a6c2e12bee710e1ab2ef031a691554ed Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 18 Apr 2021 22:56:35 +0200 Subject: [PATCH 58/58] Try to speed up and optimize some unit tests. --- st2actions/tests/unit/test_scheduler.py | 4 +- st2api/st2api/controllers/v1/packs.py | 3 +- .../tests/unit/controllers/v1/test_packs.py | 2 - .../services/test_workflow_service_retries.py | 5 ++ st2common/tests/unit/test_db.py | 61 +++++++++++++++---- .../st2reactor/container/sensor_wrapper.py | 5 ++ st2reactor/tests/unit/test_sensor_wrapper.py | 2 + .../tests/unit/controllers/v1/test_stream.py | 2 +- .../v1/test_stream_execution_output.py | 8 +-- 9 files changed, 70 insertions(+), 22 deletions(-) diff --git a/st2actions/tests/unit/test_scheduler.py b/st2actions/tests/unit/test_scheduler.py index 1a7d4b9beb..1a2d63bc1c 100644 --- a/st2actions/tests/unit/test_scheduler.py +++ b/st2actions/tests/unit/test_scheduler.py @@ -122,7 +122,7 @@ def test_next_execution(self): self.reset() schedule_q_dbs = [] - delays = [2000, 5000, 4000] + delays = [500, 1000, 800] expected_order = [0, 2, 1] test_cases = [] @@ -151,7 +151,7 @@ def test_next_execution(self): ) # Wait maximum delay seconds so the query works as expected - eventlet.sleep(3.2) + eventlet.sleep(1.0) for index in expected_order: test_case = test_cases[index] diff --git a/st2api/st2api/controllers/v1/packs.py b/st2api/st2api/controllers/v1/packs.py index 5b05ae268b..7aacae7471 100644 --- a/st2api/st2api/controllers/v1/packs.py +++ b/st2api/st2api/controllers/v1/packs.py @@ -186,7 +186,8 @@ def post(self, pack_register_request): result["policy_types"] = policies_registrar.register_policy_types(st2common) use_pack_cache = False - + # TODO: To speed up this operation since it's mostli IO bound we could use green thread + # pool here and register different resources concurrently fail_on_failure = getattr(pack_register_request, "fail_on_failure", True) for type, (Registrar, name) in six.iteritems(ENTITIES): if type in types or name in types: diff --git a/st2api/tests/unit/controllers/v1/test_packs.py b/st2api/tests/unit/controllers/v1/test_packs.py index eaaf3e3afd..111101eb65 100644 --- a/st2api/tests/unit/controllers/v1/test_packs.py +++ b/st2api/tests/unit/controllers/v1/test_packs.py @@ -502,8 +502,6 @@ def test_packs_register_endpoint(self, mock_get_packs): packs_base_path = os.path.join(fixtures_base_path, "packs") pack_names = [ "dummy_pack_1", - "dummy_pack_2", - "dummy_pack_3", "dummy_pack_10", ] mock_return_value = {} diff --git a/st2common/tests/unit/services/test_workflow_service_retries.py b/st2common/tests/unit/services/test_workflow_service_retries.py index f39315a308..8f33130d34 100644 --- a/st2common/tests/unit/services/test_workflow_service_retries.py +++ b/st2common/tests/unit/services/test_workflow_service_retries.py @@ -15,6 +15,11 @@ from __future__ import absolute_import +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + + import mock import mongoengine import os diff --git a/st2common/tests/unit/test_db.py b/st2common/tests/unit/test_db.py index 6268bb3dc9..7fe10ef3f4 100644 --- a/st2common/tests/unit/test_db.py +++ b/st2common/tests/unit/test_db.py @@ -132,7 +132,12 @@ def test_network_level_compression(self): # If running version < MongoDB 4.2 we skip this check since zstd is only supported in server # >= 4.2 - connection = db_setup(db_name=db_name, db_host=db_host, db_port=db_port) + connection = db_setup( + db_name=db_name, + db_host=db_host, + db_port=db_port, + ensure_indexes=False, + ) server_version = tuple( [int(x) for x in connection.server_info()["version"].split(".")] ) @@ -144,7 +149,12 @@ def test_network_level_compression(self): disconnect() # 1. Verify default is no compression - connection = db_setup(db_name=db_name, db_host=db_host, db_port=db_port) + connection = db_setup( + db_name=db_name, + db_host=db_host, + db_port=db_port, + ensure_indexes=False, + ) # Sadly there is no nicer way to assert that it seems self.assertFalse("compressors=['zstd']" in str(connection)) self.assertFalse("compressors" in str(connection)) @@ -154,7 +164,12 @@ def test_network_level_compression(self): cfg.CONF.set_override(name="compressors", group="database", override="zstd") - connection = db_setup(db_name=db_name, db_host=db_host, db_port=db_port) + connection = db_setup( + db_name=db_name, + db_host=db_host, + db_port=db_port, + ensure_indexes=False, + ) # Sadly there is no nicer way to assert that it seems self.assertTrue("compressors=['zstd']" in str(connection)) @@ -164,7 +179,12 @@ def test_network_level_compression(self): cfg.CONF.set_override(name="compressors", group="database", override=None) db_host = "mongodb://127.0.0.1/?compressors=zstd" - connection = db_setup(db_name=db_name, db_host=db_host, db_port=db_port) + connection = db_setup( + db_name=db_name, + db_host=db_host, + db_port=db_port, + ensure_indexes=False, + ) # Sadly there is no nicer way to assert that it seems self.assertTrue("compressors=['zstd']" in str(connection)) @@ -176,7 +196,12 @@ def test_network_level_compression(self): name="zlib_compression_level", group="database", override=8 ) - connection = db_setup(db_name=db_name, db_host=db_host, db_port=db_port) + connection = db_setup( + db_name=db_name, + db_host=db_host, + db_port=db_port, + ensure_indexes=False, + ) # Sadly there is no nicer way to assert that it seems self.assertTrue("compressors=['zlib']" in str(connection)) self.assertTrue("zlibcompressionlevel=8" in str(connection)) @@ -190,7 +215,12 @@ def test_network_level_compression(self): ) db_host = "mongodb://127.0.0.1/?compressors=zlib&zlibCompressionLevel=9" - connection = db_setup(db_name=db_name, db_host=db_host, db_port=db_port) + connection = db_setup( + db_name=db_name, + db_host=db_host, + db_port=db_port, + ensure_indexes=False, + ) # Sadly there is no nicer way to assert that it seems self.assertTrue("compressors=['zlib']" in str(connection)) self.assertTrue("zlibcompressionlevel=9" in str(connection)) @@ -283,6 +313,7 @@ def test_db_setup(self, mock_mongoengine): username="username", password="password", authentication_mechanism="MONGODB-X509", + ensure_indexes=False, ) call_args = mock_mongoengine.connection.connect.call_args_list[0][0] @@ -324,6 +355,7 @@ def test_db_setup_connecting_info_logging(self, mock_log, mock_mongoengine): db_port=db_port, username=username, password=password, + ensure_indexes=False, ) expected_message = ( @@ -350,6 +382,7 @@ def test_db_setup_connecting_info_logging(self, mock_log, mock_mongoengine): db_port=db_port, username=username, password=password, + ensure_indexes=False, ) expected_message = ( @@ -376,6 +409,7 @@ def test_db_setup_connecting_info_logging(self, mock_log, mock_mongoengine): db_port=db_port, username=username, password=password, + ensure_indexes=False, ) expected_message = ( @@ -401,6 +435,7 @@ def test_db_setup_connecting_info_logging(self, mock_log, mock_mongoengine): db_port=db_port, username=username, password=password, + ensure_indexes=False, ) expected_message = ( @@ -426,6 +461,7 @@ def test_db_setup_connecting_info_logging(self, mock_log, mock_mongoengine): db_port=db_port, username=username, password=password, + ensure_indexes=False, ) expected_message = ( @@ -464,6 +500,7 @@ def test_db_setup_connecting_info_logging(self, mock_log, mock_mongoengine): db_port=db_port, username=username, password=password, + ensure_indexes=False, ) expected_message = ( @@ -488,9 +525,7 @@ def test_db_connect_server_selection_timeout_ssl_on_non_ssl_listener(self): db_host = "localhost" db_port = 27017 - cfg.CONF.set_override( - name="connection_timeout", group="database", override=1000 - ) + cfg.CONF.set_override(name="connection_timeout", group="database", override=300) start = time.time() self.assertRaises( @@ -500,15 +535,16 @@ def test_db_connect_server_selection_timeout_ssl_on_non_ssl_listener(self): db_host=db_host, db_port=db_port, ssl=True, + ensure_indexes=False, ) end = time.time() diff = end - start - self.assertTrue(diff >= 1) + self.assertTrue(diff >= 0.3) disconnect() - cfg.CONF.set_override(name="connection_timeout", group="database", override=400) + cfg.CONF.set_override(name="connection_timeout", group="database", override=200) start = time.time() self.assertRaises( @@ -518,11 +554,12 @@ def test_db_connect_server_selection_timeout_ssl_on_non_ssl_listener(self): db_host=db_host, db_port=db_port, ssl=True, + ensure_indexes=False, ) end = time.time() diff = end - start - self.assertTrue(diff >= 0.4) + self.assertTrue(diff >= 0.1) class DbCleanupTestCase(DbTestCase): diff --git a/st2reactor/st2reactor/container/sensor_wrapper.py b/st2reactor/st2reactor/container/sensor_wrapper.py index 6c3d172a81..951052b7e3 100644 --- a/st2reactor/st2reactor/container/sensor_wrapper.py +++ b/st2reactor/st2reactor/container/sensor_wrapper.py @@ -169,6 +169,7 @@ def __init__( trigger_types, poll_interval=None, parent_args=None, + db_ensure_indexes=True, ): """ :param pack: Name of the pack this sensor belongs to. @@ -189,6 +190,9 @@ def __init__( :param parent_args: Command line arguments passed to the parent process. :type parse_args: ``list`` + + :param db_ensure_indexes: True to ensure indexes. This should really only be set to False + in tests to speed things up. """ self._pack = pack self._file_path = file_path @@ -224,6 +228,7 @@ def __init__( cfg.CONF.database.port, username=username, password=password, + ensure_indexes=db_ensure_indexes, ssl=cfg.CONF.database.ssl, ssl_keyfile=cfg.CONF.database.ssl_keyfile, ssl_certfile=cfg.CONF.database.ssl_certfile, diff --git a/st2reactor/tests/unit/test_sensor_wrapper.py b/st2reactor/tests/unit/test_sensor_wrapper.py index b2d637812d..7d7ce7f1d2 100644 --- a/st2reactor/tests/unit/test_sensor_wrapper.py +++ b/st2reactor/tests/unit/test_sensor_wrapper.py @@ -121,6 +121,7 @@ def test_sensor_creation_passive(self): class_name="TestSensor", trigger_types=trigger_types, parent_args=parent_args, + db_ensure_indexes=False, ) self.assertIsInstance(wrapper._sensor_instance, Sensor) self.assertIsNotNone(wrapper._sensor_instance) @@ -137,6 +138,7 @@ def test_sensor_creation_active(self): trigger_types=trigger_types, parent_args=parent_args, poll_interval=poll_interval, + db_ensure_indexes=False, ) self.assertIsNotNone(wrapper._sensor_instance) self.assertIsInstance(wrapper._sensor_instance, PollingSensor) diff --git a/st2stream/tests/unit/controllers/v1/test_stream.py b/st2stream/tests/unit/controllers/v1/test_stream.py index c67f3e2782..dbfb6277c1 100644 --- a/st2stream/tests/unit/controllers/v1/test_stream.py +++ b/st2stream/tests/unit/controllers/v1/test_stream.py @@ -136,7 +136,7 @@ def test_get_all(self): @mock.patch.object(st2common.stream.listener, "listen", mock.Mock()) def test_get_all_with_filters(self): - cfg.CONF.set_override(name="heartbeat", group="stream", override=0.1) + cfg.CONF.set_override(name="heartbeat", group="stream", override=0.02) listener = st2common.stream.listener.get_listener(name="stream") process_execution = listener.processor(ActionExecutionAPI) diff --git a/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py b/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py index d14dd029e8..9a135d1789 100644 --- a/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py +++ b/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py @@ -44,11 +44,11 @@ def test_get_one_id_last_no_executions_in_the_database(self): ) def test_get_output_running_execution(self): - # Retrieve lister instance to avoid race with listener connection not being established + # Retrieve listener instance to avoid race with listener connection not being established # early enough for tests to pass. # NOTE: This only affects tests where listeners are not pre-initialized. listener = get_listener(name="execution_output") - eventlet.sleep(1.0) + eventlet.sleep(0.5) # Test the execution output API endpoint for execution which is running (blocking) status = action_constants.LIVEACTION_STATUS_RUNNING @@ -89,14 +89,14 @@ def publish_action_finished(action_execution_db): output_db = ActionExecutionOutputDB(**output_params) ActionExecutionOutput.add_or_update(output_db) - eventlet.sleep(1.0) + eventlet.sleep(0.5) # Transition execution to completed state so the connection closes action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED action_execution_db = ActionExecution.add_or_update(action_execution_db) eventlet.spawn_after(0.2, insert_mock_data) - eventlet.spawn_after(1.5, publish_action_finished, action_execution_db) + eventlet.spawn_after(1.0, publish_action_finished, action_execution_db) # Retrieve data while execution is running - endpoint return new data once it's available # and block until the execution finishes