diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d22dd05789..a90bcb4e45 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -55,6 +55,12 @@ jobs: - name: 'Compile (pip deps, pylint, etc.)' task: 'ci-compile' python-version: '3.6' + - name: 'Lint Checks (black, flake8, etc.)' + task: 'ci-checks' + python-version: '3.8' + - name: 'Compile (pip deps, pylint, etc.)' + task: 'ci-compile' + python-version: '3.8' env: TASK: '${{ matrix.task }}' @@ -80,9 +86,9 @@ jobs: # TODO: maybe make the virtualenv a partial cache to exclude st2*? # !virtualenv/lib/python*/site-packages/st2* # !virtualenv/bin/st2* - key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} + key: ${{ runner.os }}-v2-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} restore-keys: | - ${{ runner.os }}-python-${{ matrix.python }}- + ${{ runner.os }}-v2-python-${{ matrix.python }}- - name: Cache APT Dependencies id: cache-apt-deps uses: actions/cache@v2 @@ -134,19 +140,24 @@ jobs: include: - name: 'Unit Tests (chunk 1)' task: 'ci-unit' - nosetests_node_total: 3 + nosetests_node_total: 2 nosetests_node_index: 0 python-version: '3.6' - name: 'Unit Tests (chunk 2)' task: 'ci-unit' - nosetests_node_total: 3 + nosetests_node_total: 2 nosetests_node_index: 1 python-version: '3.6' - - name: 'Unit Tests (chunk 3)' + - name: 'Unit Tests (chunk 1)' task: 'ci-unit' - nosetests_node_total: 3 - nosetests_node_index: 2 - python-version: '3.6' + nosetests_node_total: 2 + nosetests_node_index: 0 + python-version: '3.8' + - name: 'Unit Tests (chunk 2)' + task: 'ci-unit' + nosetests_node_total: 2 + nosetests_node_index: 1 + python-version: '3.8' # This job is slow so we only run in on a daily basis # - name: 'Micro Benchmarks' # task: 'micro-benchmarks' @@ -155,7 +166,7 @@ jobs: # nosetests_node_ index: 0 services: mongo: - image: mongo:4.0 + image: mongo:4.4 ports: - 27017:27017 @@ -309,18 +320,24 @@ jobs: nosetests_node_total: 2 nosetests_node_index: 1 python-version: '3.6' - # There are a bunch of race conditions in Orquesta tests which causes - # the step to either fail or timeout quite often. We run those tests - # separately so we can re-run just that set of tests on failure / - # timeout instead of needing to re runn the whole build. - #- name: 'Integration Tests (Orquesta)' - # task: 'ci-orquesta' - # nosetests_node_total: 1 - # nosetests_node_index: 0 - # python-version: '3.6' + - name: 'Pack Tests' + task: 'ci-packs-tests' + nosetests_node_total: 1 + nosetests_node_index: 0 + python-version: '3.8' + - name: 'Integration Tests (chunk 1)' + task: 'ci-integration' + nosetests_node_total: 2 + nosetests_node_index: 0 + python-version: '3.8' + - name: 'Integration Tests (chunk 2)' + task: 'ci-integration' + nosetests_node_total: 2 + nosetests_node_index: 1 + python-version: '3.8' services: mongo: - image: mongo:4.0 + image: mongo:4.4 ports: - 27017:27017 @@ -466,13 +483,14 @@ jobs: ./scripts/github/configure-rabbitmq.sh - name: Print versions run: | + ./scripts/ci/print-versions.sh - name: make if: "${{ env.TASK == 'ci-integration' }}" #timeout-minutes: 7 # TODO: Use dynamic timeout value based on the branch - for master we # need to use timeout x2 due to coverage overhead - timeout-minutes: 14 + timeout-minutes: 14 # may die if rabbitmq fails to start # use: script -e -c to print colors run: | script -e -c "make ${TASK}" && exit 0 diff --git a/.github/workflows/orquesta-integration-tests.yaml b/.github/workflows/orquesta-integration-tests.yaml index 26477b26be..7f520b41e2 100644 --- a/.github/workflows/orquesta-integration-tests.yaml +++ b/.github/workflows/orquesta-integration-tests.yaml @@ -55,9 +55,14 @@ jobs: nosetests_node_total: 1 nosetests_node_index: 0 python-version: '3.6' + - name: 'Integration Tests (Orquesta)' + task: 'ci-orquesta' + nosetests_node_total: 1 + nosetests_node_index: 0 + python-version: '3.8' services: mongo: - image: mongo:4.0 + image: mongo:4.4 ports: - 27017:27017 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c539e0b854..bdd3a3aee6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,9 @@ # files. # This hook relies on development virtual environment being present in virtualenv/. default_language_version: - python: python3.6 + # We don't specify a specific version so it uses either Python 3.6 / Python + # 3.8 depending on what is available locally and on the runner + python: python exclude: '(build|dist)' repos: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 18d0240a88..a79a2b042a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -49,6 +49,22 @@ Added Contributed by @cognifloyd. +* Add new ``database.compressors`` and ``database.zlib_compression_level`` config option which + specifies compression algorithms client supports for network / transport level compression + when talking to MongoDB. + + Actual compression algorithm used will be then decided by the server and depends on the + algorithms which are supported by the server + client. + + Possible / valid values include: zstd, zlib. Keep in mind that zstandard (zstd) is only supported + by MongoDB >= 4.2. + + Our official Debian and RPM packages bundle ``zstandard`` dependency by default which means + setting this value to ``zstd`` should work out of the box as long as the server runs + MongoDB >= 4.2. #5177 + + Contributed by @Kami. + Changed ~~~~~~~ @@ -172,6 +188,10 @@ Changed Contributed by @Kami. +* Update code and dependencies so it supports Python 3.8 and Mongo DB 4.4 #5177 + + Contributed by @nzloshm @winem @Kami. + * StackStorm Web UI (``st2web``) has been updated to not render and display execution results larger than 200 KB directly in the history panel in the right side bar by default anymore. Instead a link to view or download the raw result is displayed. diff --git a/Makefile b/Makefile index fdf935632c..1417024255 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) SHELL := /bin/bash -TOX_DIR := .tox OS := $(shell uname) # We separate the OSX X and Linux virtualenvs so we can run in a Docker diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index 488939eb55..97885cc5a7 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -113,6 +113,8 @@ url = None [database] # Specifies database authentication mechanisms. By default, it use SCRAM-SHA-1 with MongoDB 3.0 and later, MONGODB-CR (MongoDB Challenge Response protocol) for older servers. authentication_mechanism = None +# Comma delimited string of compression algorithms to use for transport level compression. Actual algorithm will then be decided based on the algorithms supported by the client and the server. For example: zstd. Defaults to no compression. Keep in mind that zstd is only supported with MongoDB 4.2 and later. +compressors = # Connection retry backoff max (seconds). connection_retry_backoff_max_s = 10 # Backoff multiplier (seconds). @@ -143,6 +145,8 @@ ssl_keyfile = None ssl_match_hostname = True # username for db login username = None +# Compression level when compressors is set to zlib. Valid calues are -1 to 9. Defaults to 6. +zlib_compression_level = [exporter] # Directory to dump data to. diff --git a/contrib/linux/actions/service.py b/contrib/linux/actions/service.py index 335e5038f6..70db65773b 100644 --- a/contrib/linux/actions/service.py +++ b/contrib/linux/actions/service.py @@ -15,6 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +NOTE: This script file utilizes remote-shell-script runner which means it copied as-is to the +remote host and executed using Python binary available on that systems. + +This means it doesn't use pack or StackStorm specific virtual environment which means we can't +rely on any 3rd party dependencies. +""" + import re import sys import os @@ -23,13 +31,33 @@ from st2common.util.shell import quote_unix -distro = platform.linux_distribution()[0] + +def get_linux_distribution(): + # platform.linux_distribution() is not available in Python >= 3.8 + if hasattr(platform, "linux_distribution"): + distro = platform.linux_distribution()[0] # pylint: disable=no-member + else: + # Fall back to shelling out to lsb_release + result = subprocess.run( + "lsb_release -i -s", shell=True, check=True, stdout=subprocess.PIPE + ) + distro = result.stdout.decode("utf-8").strip() + + if not distro: + raise ValueError("Fail to detect distribution we are running on") + + return distro + if len(sys.argv) < 3: raise ValueError("Usage: service.py ") +distro = get_linux_distribution() + args = {"act": quote_unix(sys.argv[1]), "service": quote_unix(sys.argv[2])} +print("Detected distro: %s" % (distro)) + if re.search(distro, "Ubuntu"): if os.path.isfile("/etc/init/%s.conf" % args["service"]): cmd_args = ["service", args["service"], args["act"]] diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py index a694ffceac..38441933b2 100644 --- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py +++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py @@ -21,6 +21,10 @@ import mock import tempfile +from st2common.util.monkey_patch import use_select_poll_workaround + +use_select_poll_workaround() + from oslo_config import cfg from python_runner import python_runner diff --git a/fixed-requirements.txt b/fixed-requirements.txt index f10fbd36b4..b11c2398d7 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -31,6 +31,7 @@ prance==0.9.0 prompt-toolkit==1.0.15 pyinotify==0.9.6; platform_system=="Linux" pymongo==3.11.3 +zstandard==0.15.2 python-editor==1.0.4 python-keyczar==0.716 pytz==2021.1 @@ -42,6 +43,10 @@ retrying==1.3.3 routes==2.4.1 semver==2.9.0 six==1.13.0 +argparse==1.12.2 +argcomplete==1.12.2 +prettytable==2.1.0 +importlib-metadata==3.10.1 # NOTE: sseclient has various issues which sometimes hang the connection for a long time, etc. sseclient-py==1.7 stevedore==1.30.1 diff --git a/requirements.txt b/requirements.txt index 1f8a7d3841..ec14b5b53e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ RandomWords amqp==2.5.2 apscheduler==3.7.0 -argcomplete +argcomplete==1.12.2 bcrypt==3.2.0 chardet<3.1.0 cryptography==3.4.7 @@ -23,6 +23,7 @@ git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backe gitpython==2.1.15 greenlet==1.0.0 gunicorn==20.1.0 +importlib-metadata==3.10.1 jinja2==2.11.3 jsonpath-rw==1.4.0 jsonschema==2.6.0 @@ -39,7 +40,7 @@ oslo.config>=1.12.1,<1.13 oslo.utils<5.0,>=4.0.0 paramiko==2.7.2 passlib==1.7.4 -prettytable +prettytable==2.1.0 prompt-toolkit==1.0.15 psutil==5.8.0 pyinotify==0.9.6; platform_system=="Linux" @@ -68,3 +69,4 @@ unittest2 webob==1.8.7 webtest zake==0.2.2 +zstandard==0.15.2 diff --git a/st2actions/tests/unit/test_scheduler.py b/st2actions/tests/unit/test_scheduler.py index 1a7d4b9beb..1a2d63bc1c 100644 --- a/st2actions/tests/unit/test_scheduler.py +++ b/st2actions/tests/unit/test_scheduler.py @@ -122,7 +122,7 @@ def test_next_execution(self): self.reset() schedule_q_dbs = [] - delays = [2000, 5000, 4000] + delays = [500, 1000, 800] expected_order = [0, 2, 1] test_cases = [] @@ -151,7 +151,7 @@ def test_next_execution(self): ) # Wait maximum delay seconds so the query works as expected - eventlet.sleep(3.2) + eventlet.sleep(1.0) for index in expected_order: test_case = test_cases[index] diff --git a/st2api/st2api/controllers/v1/packs.py b/st2api/st2api/controllers/v1/packs.py index 5b05ae268b..7aacae7471 100644 --- a/st2api/st2api/controllers/v1/packs.py +++ b/st2api/st2api/controllers/v1/packs.py @@ -186,7 +186,8 @@ def post(self, pack_register_request): result["policy_types"] = policies_registrar.register_policy_types(st2common) use_pack_cache = False - + # TODO: To speed up this operation since it's mostli IO bound we could use green thread + # pool here and register different resources concurrently fail_on_failure = getattr(pack_register_request, "fail_on_failure", True) for type, (Registrar, name) in six.iteritems(ENTITIES): if type in types or name in types: diff --git a/st2api/tests/unit/controllers/v1/test_executions_auth.py b/st2api/tests/unit/controllers/v1/test_executions_auth.py index 9f3842d4d5..9baaf1e0f3 100644 --- a/st2api/tests/unit/controllers/v1/test_executions_auth.py +++ b/st2api/tests/unit/controllers/v1/test_executions_auth.py @@ -130,7 +130,7 @@ class ActionExecutionControllerTestCaseAuthEnabled(FunctionalTest): @classmethod @mock.patch.object(Token, "get", mock.MagicMock(side_effect=mock_get_token)) - @mock.patch.object(User, "get_by_name", mock_get_by_name) + @mock.patch.object(User, "get_by_name", mock.MagicMock(return_value=TEST_USER)) @mock.patch.object( action_validator, "validate_action", mock.MagicMock(return_value=True) ) diff --git a/st2api/tests/unit/controllers/v1/test_packs.py b/st2api/tests/unit/controllers/v1/test_packs.py index 07cacd0be8..111101eb65 100644 --- a/st2api/tests/unit/controllers/v1/test_packs.py +++ b/st2api/tests/unit/controllers/v1/test_packs.py @@ -17,6 +17,7 @@ import requests import mock +import sys from st2common.content.loader import ContentPackLoader from st2common.models.db.pack import PackDB @@ -345,6 +346,13 @@ def test_index_health(self): def test_index_health_broken(self): resp = self.app.get("/v1/packs/index/health") + # up to Py3.6 requests.exceptions.RequestException() appends a trailing , + if sys.version_info < (3, 8): + broken_index_message = "RequestException('index is broken',)" + else: + broken_index_message = "RequestException('index is broken')" + + self.maxDiff = None self.assertEqual(resp.status_int, 200) self.assertEqual( resp.json, @@ -361,7 +369,7 @@ def test_index_health_broken(self): }, { "url": "http://broken.example.com", - "message": "RequestException('index is broken',)", + "message": broken_index_message, "packs": 0, "error": "unresponsive", }, @@ -494,8 +502,6 @@ def test_packs_register_endpoint(self, mock_get_packs): packs_base_path = os.path.join(fixtures_base_path, "packs") pack_names = [ "dummy_pack_1", - "dummy_pack_2", - "dummy_pack_3", "dummy_pack_10", ] mock_return_value = {} diff --git a/st2client/in-requirements.txt b/st2client/in-requirements.txt index 6d1341e742..da020c46df 100644 --- a/st2client/in-requirements.txt +++ b/st2client/in-requirements.txt @@ -1,4 +1,5 @@ # Remember to list implicit packages here, otherwise version won't be fixated! +importlib-metadata argcomplete prettytable pytz diff --git a/st2client/requirements.txt b/st2client/requirements.txt index 5103232989..f8d6c9ba98 100644 --- a/st2client/requirements.txt +++ b/st2client/requirements.txt @@ -5,13 +5,14 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -argcomplete +argcomplete==1.12.2 chardet<3.1.0 cryptography==3.4.7 +importlib-metadata==3.10.1 jsonpath-rw==1.4.0 jsonschema==2.6.0 orjson==3.5.1 -prettytable +prettytable==2.1.0 prompt-toolkit==1.0.15 python-dateutil==2.8.1 python-editor==1.0.4 diff --git a/st2common/in-requirements.txt b/st2common/in-requirements.txt index c427e34614..9307d21057 100644 --- a/st2common/in-requirements.txt +++ b/st2common/in-requirements.txt @@ -16,6 +16,8 @@ oslo.config paramiko pyyaml pymongo +# used for optional network level compression for mongodb +zstandard cryptography requests retrying diff --git a/st2common/requirements.txt b/st2common/requirements.txt index 78ae5b9a9e..8e78c26d1e 100644 --- a/st2common/requirements.txt +++ b/st2common/requirements.txt @@ -40,3 +40,4 @@ tooz==2.8.0 udatetime==0.0.16 webob==1.8.7 zake==0.2.2 +zstandard==0.15.2 diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index 0429c81ea2..9d39f86d64 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -243,6 +243,20 @@ def register_opts(ignore_errors=False): "By default, it use SCRAM-SHA-1 with MongoDB 3.0 and later, " "MONGODB-CR (MongoDB Challenge Response protocol) for older servers.", ), + cfg.StrOpt( + "compressors", + default="", + help="Comma delimited string of compression algorithms to use for transport level " + "compression. Actual algorithm will then be decided based on the algorithms " + "supported by the client and the server. For example: zstd. Defaults to no " + "compression. Keep in mind that zstd is only supported with MongoDB 4.2 and later.", + ), + cfg.IntOpt( + "zlib_compression_level", + default="", + help="Compression level when compressors is set to zlib. Valid calues are -1 to 9. " + "Defaults to 6.", + ), ] do_register_opts(db_opts, "database", ignore_errors) diff --git a/st2common/st2common/models/db/__init__.py b/st2common/st2common/models/db/__init__.py index ee7261facd..9ed99c7c91 100644 --- a/st2common/st2common/models/db/__init__.py +++ b/st2common/st2common/models/db/__init__.py @@ -15,6 +15,32 @@ from __future__ import absolute_import +import sys + +# NOTE: We need to perform eventlet monkey patching, especially the thread module before importing +# pymongo and mongoengine. If we don't do that, tests will hang because pymongo connection checker +# thread will be constructed before monkey patching with reference to non patched threading module. +# This is not an issue for any service where we have code structured correctly so we perform +# monkey patching as early as possible, but this is not always the case with tests and when monkey +# patching happens inside the tests really depends on tests import ordering, etc. +# +# One option would be to simply add monkey_patch() call to the top of every single test file, but +# this would result in tons of duplication. +# +# Another option is to simply perform monkey patching right here before importing pymongo in case +# we detected we are running inside tests. +# +# And third option is to re-arrange the imports so we lazily import pymongo + mongoengine when we +# first need it because monkey patching will already be performed by then. +# +# For now, we go with option 2) since it seems to be good enough of a compromise. We detect if we +# are running inside tests by checking if "nose" module is present - the same logic we already use +# in a couple of other places (and something which would need to be changed if we switch to pytest). +if "nose" in sys.modules.keys(): + from st2common.util.monkey_patch import monkey_patch + + monkey_patch() + import copy import importlib import traceback @@ -141,6 +167,16 @@ def _db_connect( ssl_match_hostname=ssl_match_hostname, ) + compressor_kwargs = {} + + if cfg.CONF.database.compressors: + compressor_kwargs["compressors"] = cfg.CONF.database.compressors + + if cfg.CONF.database.zlib_compression_level is not None: + compressor_kwargs[ + "zlibCompressionLevel" + ] = cfg.CONF.database.zlib_compression_level + # NOTE: We intentionally set "serverSelectionTimeoutMS" to 3 seconds. By default it's set to # 30 seconds, which means it will block up to 30 seconds and fail if there are any SSL related # or other errors @@ -155,6 +191,7 @@ def _db_connect( connectTimeoutMS=connection_timeout, serverSelectionTimeoutMS=connection_timeout, **ssl_kwargs, + **compressor_kwargs, ) # NOTE: Since pymongo 3.0, connect() method is lazy and not blocking (always returns success) diff --git a/st2common/st2common/service_setup.py b/st2common/st2common/service_setup.py index cb3e1433f9..44bfc3d4da 100644 --- a/st2common/st2common/service_setup.py +++ b/st2common/st2common/service_setup.py @@ -136,6 +136,7 @@ def setup( lang_env = os.environ.get("LANG", "unknown") lang_env = os.environ.get("LANG", "notset") pythonioencoding_env = os.environ.get("PYTHONIOENCODING", "notset") + try: language_code, encoding = locale.getdefaultlocale() @@ -145,6 +146,7 @@ def setup( used_locale = "unable to retrieve locale" except Exception as e: used_locale = "unable to retrieve locale: %s " % (str(e)) + encoding = "unknown" LOG.info("Using Python: %s (%s)" % (version, sys.executable)) LOG.info( diff --git a/st2common/st2common/util/keyvalue.py b/st2common/st2common/util/keyvalue.py index cad32250a8..1fc538025f 100644 --- a/st2common/st2common/util/keyvalue.py +++ b/st2common/st2common/util/keyvalue.py @@ -94,7 +94,7 @@ def get_key(key=None, user_db=None, scope=None, decrypt=False): if not user_db: # Use system user - user_db = UserDB(cfg.CONF.system_user.user) + user_db = UserDB(name=cfg.CONF.system_user.user) scope, key_id = _derive_scope_and_key(key=key, user=user_db.name, scope=scope) scope = get_datastore_full_scope(scope) diff --git a/st2common/st2common/util/pack_management.py b/st2common/st2common/util/pack_management.py index 0fde5b1d86..12d0251e00 100644 --- a/st2common/st2common/util/pack_management.py +++ b/st2common/st2common/util/pack_management.py @@ -26,6 +26,11 @@ import stat import re +# This test workaround needs to be used before importing git +from st2common.util.monkey_patch import use_select_poll_workaround + +use_select_poll_workaround() + import six from git.repo import Repo from gitdb.exc import BadName, BadObject diff --git a/st2common/st2common/util/service.py b/st2common/st2common/util/service.py index e3c2dcb9f9..3171eb9983 100644 --- a/st2common/st2common/util/service.py +++ b/st2common/st2common/util/service.py @@ -15,8 +15,6 @@ from __future__ import absolute_import -import pymongo - from st2common import log as logging @@ -24,6 +22,8 @@ def retry_on_exceptions(exc): + import pymongo + LOG.warning("Evaluating retry on exception %s. %s", type(exc), str(exc)) is_mongo_connection_error = isinstance(exc, pymongo.errors.ConnectionFailure) diff --git a/st2common/tests/integration/test_logging.py b/st2common/tests/integration/test_logging.py index af4eab3bf1..997f96c7a5 100644 --- a/st2common/tests/integration/test_logging.py +++ b/st2common/tests/integration/test_logging.py @@ -18,6 +18,7 @@ import os import sys import signal +import unittest import eventlet from eventlet.green import subprocess @@ -30,7 +31,9 @@ class LogFormattingAndEncodingTestCase(IntegrationTestCase): - def test_formatting_with_unicode_data_works_no_stdout_patching(self): + def test_formatting_with_unicode_data_works_no_stdout_patching_valid_utf8_encoding( + self, + ): # Ensure that process doesn't end up in an infinite loop if non-utf8 locale / encoding is # used and a unicode sequence is logged. @@ -66,6 +69,13 @@ def test_formatting_with_unicode_data_works_no_stdout_patching(self): "DEBUG [-] Test debug message with unicode 1 - \u597d\u597d\u597d", stdout ) + @unittest.skipIf(sys.version_info >= (3, 8, 0), "Skipping test under Python >= 3.8") + def test_formatting_with_unicode_data_works_no_stdout_patching_non_valid_utf8_encoding( + self, + ): + # Ensure that process doesn't end up in an infinite loop if non-utf8 locale / encoding is + # used and a unicode sequence is logged. + # 2. Process is not using utf-8 encoding - LC_ALL set to invalid locale - should result in # single exception being logged, but not infinite loop process = self._start_process( @@ -105,6 +115,12 @@ def test_formatting_with_unicode_data_works_no_stdout_patching(self): "DEBUG [-] Test debug message with unicode 1 - \u597d\u597d\u597d", stdout ) + def test_formatting_with_unicode_data_works_no_stdout_patching_ascii_pythonioencoding( + self, + ): + # Ensure that process doesn't end up in an infinite loop if non-utf8 locale / encoding is + # used and a unicode sequence is logged. + # 3. Process is not using utf-8 encoding - PYTHONIOENCODING set to ascii - should result in # single exception being logged, but not infinite loop process = self._start_process( @@ -144,7 +160,9 @@ def test_formatting_with_unicode_data_works_no_stdout_patching(self): "DEBUG [-] Test debug message with unicode 1 - \u597d\u597d\u597d", stdout ) - def test_formatting_with_unicode_data_works_with_stdout_patching(self): + def test_formatting_with_unicode_data_works_with_stdout_patching_valid_utf8_encoding( + self, + ): # Test a scenario where patching is enabled which means it should never result in infinite # loop # 1. Process is using a utf-8 encoding @@ -164,7 +182,6 @@ def test_formatting_with_unicode_data_works_with_stdout_patching(self): stdout = process.stdout.read().decode("utf-8") stderr = process.stderr.read().decode("utf-8") stdout_lines = stdout.split("\n") - print(stderr) self.assertEqual(stderr, "") self.assertTrue(len(stdout_lines) < 20) @@ -180,6 +197,9 @@ def test_formatting_with_unicode_data_works_with_stdout_patching(self): "DEBUG [-] Test debug message with unicode 1 - \u597d\u597d\u597d", stdout ) + def test_formatting_with_unicode_data_works_with_stdout_patching_non_valid_utf8_encoding( + self, + ): # 2. Process is not using utf-8 encoding process = self._start_process( env={ @@ -196,9 +216,11 @@ def test_formatting_with_unicode_data_works_with_stdout_patching(self): stdout = process.stdout.read().decode("utf-8") stderr = process.stderr.read().decode("utf-8") + stdout_lines = stdout.split("\n") self.assertEqual(stderr, "") - self.assertTrue(len(stdout_lines) < 50) + print(stdout) + self.assertTrue(len(stdout_lines) < 100) self.assertIn("INFO [-] Test info message 1", stdout) self.assertIn("Test debug message 1", stdout) @@ -211,6 +233,9 @@ def test_formatting_with_unicode_data_works_with_stdout_patching(self): "DEBUG [-] Test debug message with unicode 1 - \u597d\u597d\u597d", stdout ) + def test_formatting_with_unicode_data_works_with_stdout_patching__ascii_pythonioencoding( + self, + ): # 3. Process is not using utf-8 encoding - PYTHONIOENCODING set to ascii process = self._start_process( env={ diff --git a/st2common/tests/integration/test_service_setup_log_level_filtering.py b/st2common/tests/integration/test_service_setup_log_level_filtering.py index 51febaa678..6b91a0874a 100644 --- a/st2common/tests/integration/test_service_setup_log_level_filtering.py +++ b/st2common/tests/integration/test_service_setup_log_level_filtering.py @@ -63,17 +63,35 @@ class ServiceSetupLogLevelFilteringTestCase(IntegrationTestCase): + def setUp(self): + super(ServiceSetupLogLevelFilteringTestCase, self).setUp() + self._reset_env() + + def tearDown(self): + super(ServiceSetupLogLevelFilteringTestCase, self).tearDown() + self._reset_env() + + def _reset_env(self): + keys_to_delete = ["LC_ALL", "ST2_LOG_PATCH_STDOUT", "PYTHONIOENCODING"] + + for key in keys_to_delete: + if key in os.environ: + del os.environ[key] + def test_system_info_is_logged_on_startup(self): # Verify INFO level service start up messages process = self._start_process(config_path=ST2_CONFIG_INFO_LL_PATH) self.add_process(process=process) # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(4) process.send_signal(signal.SIGKILL) # Verify first 4 environment related log messages stdout = process.stdout.read().decode("utf-8") + stderr = process.stderr.read().decode("utf-8") + print(stdout) + print(stderr) self.assertIn("INFO [-] Using Python:", stdout) self.assertIn("INFO [-] Using fs encoding:", stdout) self.assertIn("INFO [-] Using config files:", stdout) @@ -82,17 +100,25 @@ def test_system_info_is_logged_on_startup(self): def test_warning_is_emitted_on_non_utf8_encoding(self): env = os.environ.copy() env["LC_ALL"] = "invalid" + env["ST2_LOG_PATCH_STDOUT"] = "false" + env["PYTHONIOENCODING"] = "ascii" process = self._start_process(config_path=ST2_CONFIG_INFO_LL_PATH, env=env) self.add_process(process=process) # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(4) process.send_signal(signal.SIGKILL) # Verify first 4 environment related log messages stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")) + stderr = process.stderr.read().decode("utf-8") + print(stdout) + print(stderr) self.assertIn("WARNING [-] Detected a non utf-8 locale / encoding", stdout) - self.assertIn("fs encoding: ascii", stdout) + + if sys.version_info < (3, 8, 0): + self.assertIn("fs encoding: ascii", stdout) + self.assertIn("unknown locale: invalid", stdout) def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): @@ -101,7 +127,7 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): self.add_process(process=process) # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(4) process.send_signal(signal.SIGKILL) # Verify first 4 environment related log messages @@ -118,7 +144,7 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): self.add_process(process=process) # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(4) process.send_signal(signal.SIGKILL) # First 6 log lines are debug messages about the environment which are always logged diff --git a/st2common/tests/unit/services/test_workflow_service_retries.py b/st2common/tests/unit/services/test_workflow_service_retries.py index f39315a308..8f33130d34 100644 --- a/st2common/tests/unit/services/test_workflow_service_retries.py +++ b/st2common/tests/unit/services/test_workflow_service_retries.py @@ -15,6 +15,11 @@ from __future__ import absolute_import +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + + import mock import mongoengine import os diff --git a/st2common/tests/unit/test_content_loader.py b/st2common/tests/unit/test_content_loader.py index 566ddf4f76..279d04dc4a 100644 --- a/st2common/tests/unit/test_content_loader.py +++ b/st2common/tests/unit/test_content_loader.py @@ -21,7 +21,7 @@ import yaml -from yaml import SafeLoader +from yaml import SafeLoader, FullLoader try: from yaml import CSafeLoader @@ -121,8 +121,8 @@ def test_yaml_safe_load(self): dumped = yaml.dump(Foo) self.assertTrue("!!python" in dumped) - # Regular load should work, but safe wrapper should fail - result = yaml.load(dumped) + # Regular full load should work, but safe wrapper should fail + result = yaml.load(dumped, Loader=FullLoader) self.assertTrue(result) self.assertRaisesRegexp( diff --git a/st2common/tests/unit/test_db.py b/st2common/tests/unit/test_db.py index da0157127e..7fe10ef3f4 100644 --- a/st2common/tests/unit/test_db.py +++ b/st2common/tests/unit/test_db.py @@ -103,10 +103,12 @@ class DbConnectionTestCase(DbTestCase): def setUp(self): # NOTE: It's important we re-establish a connection on each setUp self.setUpClass() + cfg.CONF.reset() def tearDown(self): # NOTE: It's important we disconnect here otherwise tests will fail disconnect() + cfg.CONF.reset() def test_check_connect(self): """ @@ -121,6 +123,108 @@ def test_check_connect(self): ) self.assertIn(expected_str, str(client), "Not connected to desired host.") + def test_network_level_compression(self): + disconnect() + + db_name = "st2" + db_host = "localhost" + db_port = 27017 + + # If running version < MongoDB 4.2 we skip this check since zstd is only supported in server + # >= 4.2 + connection = db_setup( + db_name=db_name, + db_host=db_host, + db_port=db_port, + ensure_indexes=False, + ) + server_version = tuple( + [int(x) for x in connection.server_info()["version"].split(".")] + ) + + if server_version < (4, 2, 0): + self.skipTest("Skipping test since running MongoDB < 4.2") + return + + disconnect() + + # 1. Verify default is no compression + connection = db_setup( + db_name=db_name, + db_host=db_host, + db_port=db_port, + ensure_indexes=False, + ) + # Sadly there is no nicer way to assert that it seems + self.assertFalse("compressors=['zstd']" in str(connection)) + self.assertFalse("compressors" in str(connection)) + + # 2. Verify using zstd works - specified using config option + disconnect() + + cfg.CONF.set_override(name="compressors", group="database", override="zstd") + + connection = db_setup( + db_name=db_name, + db_host=db_host, + db_port=db_port, + ensure_indexes=False, + ) + # Sadly there is no nicer way to assert that it seems + self.assertTrue("compressors=['zstd']" in str(connection)) + + # 3. Verify using zstd works - specified inside URI + disconnect() + + cfg.CONF.set_override(name="compressors", group="database", override=None) + db_host = "mongodb://127.0.0.1/?compressors=zstd" + + connection = db_setup( + db_name=db_name, + db_host=db_host, + db_port=db_port, + ensure_indexes=False, + ) + # Sadly there is no nicer way to assert that it seems + self.assertTrue("compressors=['zstd']" in str(connection)) + + # 4. Verify using zlib works - specified using config option + disconnect() + + cfg.CONF.set_override(name="compressors", group="database", override="zlib") + cfg.CONF.set_override( + name="zlib_compression_level", group="database", override=8 + ) + + connection = db_setup( + db_name=db_name, + db_host=db_host, + db_port=db_port, + ensure_indexes=False, + ) + # Sadly there is no nicer way to assert that it seems + self.assertTrue("compressors=['zlib']" in str(connection)) + self.assertTrue("zlibcompressionlevel=8" in str(connection)) + + # 5. Verify using zlib works - specified inside URI + disconnect() + + cfg.CONF.set_override(name="compressors", group="database", override=None) + cfg.CONF.set_override( + name="zlib_compression_level", group="database", override=None + ) + db_host = "mongodb://127.0.0.1/?compressors=zlib&zlibCompressionLevel=9" + + connection = db_setup( + db_name=db_name, + db_host=db_host, + db_port=db_port, + ensure_indexes=False, + ) + # Sadly there is no nicer way to assert that it seems + self.assertTrue("compressors=['zlib']" in str(connection)) + self.assertTrue("zlibcompressionlevel=9" in str(connection)) + def test_get_ssl_kwargs(self): # 1. No SSL kwargs provided ssl_kwargs = _get_ssl_kwargs() @@ -209,6 +313,7 @@ def test_db_setup(self, mock_mongoengine): username="username", password="password", authentication_mechanism="MONGODB-X509", + ensure_indexes=False, ) call_args = mock_mongoengine.connection.connect.call_args_list[0][0] @@ -250,6 +355,7 @@ def test_db_setup_connecting_info_logging(self, mock_log, mock_mongoengine): db_port=db_port, username=username, password=password, + ensure_indexes=False, ) expected_message = ( @@ -276,6 +382,7 @@ def test_db_setup_connecting_info_logging(self, mock_log, mock_mongoengine): db_port=db_port, username=username, password=password, + ensure_indexes=False, ) expected_message = ( @@ -302,6 +409,7 @@ def test_db_setup_connecting_info_logging(self, mock_log, mock_mongoengine): db_port=db_port, username=username, password=password, + ensure_indexes=False, ) expected_message = ( @@ -327,6 +435,7 @@ def test_db_setup_connecting_info_logging(self, mock_log, mock_mongoengine): db_port=db_port, username=username, password=password, + ensure_indexes=False, ) expected_message = ( @@ -352,6 +461,7 @@ def test_db_setup_connecting_info_logging(self, mock_log, mock_mongoengine): db_port=db_port, username=username, password=password, + ensure_indexes=False, ) expected_message = ( @@ -390,6 +500,7 @@ def test_db_setup_connecting_info_logging(self, mock_log, mock_mongoengine): db_port=db_port, username=username, password=password, + ensure_indexes=False, ) expected_message = ( @@ -414,9 +525,7 @@ def test_db_connect_server_selection_timeout_ssl_on_non_ssl_listener(self): db_host = "localhost" db_port = 27017 - cfg.CONF.set_override( - name="connection_timeout", group="database", override=1000 - ) + cfg.CONF.set_override(name="connection_timeout", group="database", override=300) start = time.time() self.assertRaises( @@ -426,15 +535,16 @@ def test_db_connect_server_selection_timeout_ssl_on_non_ssl_listener(self): db_host=db_host, db_port=db_port, ssl=True, + ensure_indexes=False, ) end = time.time() diff = end - start - self.assertTrue(diff >= 1) + self.assertTrue(diff >= 0.3) disconnect() - cfg.CONF.set_override(name="connection_timeout", group="database", override=400) + cfg.CONF.set_override(name="connection_timeout", group="database", override=200) start = time.time() self.assertRaises( @@ -444,11 +554,12 @@ def test_db_connect_server_selection_timeout_ssl_on_non_ssl_listener(self): db_host=db_host, db_port=db_port, ssl=True, + ensure_indexes=False, ) end = time.time() diff = end - start - self.assertTrue(diff >= 0.4) + self.assertTrue(diff >= 0.1) class DbCleanupTestCase(DbTestCase): diff --git a/st2reactor/st2reactor/container/sensor_wrapper.py b/st2reactor/st2reactor/container/sensor_wrapper.py index 6c3d172a81..951052b7e3 100644 --- a/st2reactor/st2reactor/container/sensor_wrapper.py +++ b/st2reactor/st2reactor/container/sensor_wrapper.py @@ -169,6 +169,7 @@ def __init__( trigger_types, poll_interval=None, parent_args=None, + db_ensure_indexes=True, ): """ :param pack: Name of the pack this sensor belongs to. @@ -189,6 +190,9 @@ def __init__( :param parent_args: Command line arguments passed to the parent process. :type parse_args: ``list`` + + :param db_ensure_indexes: True to ensure indexes. This should really only be set to False + in tests to speed things up. """ self._pack = pack self._file_path = file_path @@ -224,6 +228,7 @@ def __init__( cfg.CONF.database.port, username=username, password=password, + ensure_indexes=db_ensure_indexes, ssl=cfg.CONF.database.ssl, ssl_keyfile=cfg.CONF.database.ssl_keyfile, ssl_certfile=cfg.CONF.database.ssl_certfile, diff --git a/st2reactor/tests/unit/test_sensor_wrapper.py b/st2reactor/tests/unit/test_sensor_wrapper.py index b2d637812d..7d7ce7f1d2 100644 --- a/st2reactor/tests/unit/test_sensor_wrapper.py +++ b/st2reactor/tests/unit/test_sensor_wrapper.py @@ -121,6 +121,7 @@ def test_sensor_creation_passive(self): class_name="TestSensor", trigger_types=trigger_types, parent_args=parent_args, + db_ensure_indexes=False, ) self.assertIsInstance(wrapper._sensor_instance, Sensor) self.assertIsNotNone(wrapper._sensor_instance) @@ -137,6 +138,7 @@ def test_sensor_creation_active(self): trigger_types=trigger_types, parent_args=parent_args, poll_interval=poll_interval, + db_ensure_indexes=False, ) self.assertIsNotNone(wrapper._sensor_instance) self.assertIsInstance(wrapper._sensor_instance, PollingSensor) diff --git a/st2stream/tests/unit/controllers/v1/test_stream.py b/st2stream/tests/unit/controllers/v1/test_stream.py index c67f3e2782..dbfb6277c1 100644 --- a/st2stream/tests/unit/controllers/v1/test_stream.py +++ b/st2stream/tests/unit/controllers/v1/test_stream.py @@ -136,7 +136,7 @@ def test_get_all(self): @mock.patch.object(st2common.stream.listener, "listen", mock.Mock()) def test_get_all_with_filters(self): - cfg.CONF.set_override(name="heartbeat", group="stream", override=0.1) + cfg.CONF.set_override(name="heartbeat", group="stream", override=0.02) listener = st2common.stream.listener.get_listener(name="stream") process_execution = listener.processor(ActionExecutionAPI) diff --git a/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py b/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py index d14dd029e8..9a135d1789 100644 --- a/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py +++ b/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py @@ -44,11 +44,11 @@ def test_get_one_id_last_no_executions_in_the_database(self): ) def test_get_output_running_execution(self): - # Retrieve lister instance to avoid race with listener connection not being established + # Retrieve listener instance to avoid race with listener connection not being established # early enough for tests to pass. # NOTE: This only affects tests where listeners are not pre-initialized. listener = get_listener(name="execution_output") - eventlet.sleep(1.0) + eventlet.sleep(0.5) # Test the execution output API endpoint for execution which is running (blocking) status = action_constants.LIVEACTION_STATUS_RUNNING @@ -89,14 +89,14 @@ def publish_action_finished(action_execution_db): output_db = ActionExecutionOutputDB(**output_params) ActionExecutionOutput.add_or_update(output_db) - eventlet.sleep(1.0) + eventlet.sleep(0.5) # Transition execution to completed state so the connection closes action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED action_execution_db = ActionExecution.add_or_update(action_execution_db) eventlet.spawn_after(0.2, insert_mock_data) - eventlet.spawn_after(1.5, publish_action_finished, action_execution_db) + eventlet.spawn_after(1.0, publish_action_finished, action_execution_db) # Retrieve data while execution is running - endpoint return new data once it's available # and block until the execution finishes diff --git a/st2tests/st2tests/base.py b/st2tests/st2tests/base.py index de90571684..f1237978b4 100644 --- a/st2tests/st2tests/base.py +++ b/st2tests/st2tests/base.py @@ -551,12 +551,20 @@ class IntegrationTestCase(TestCase): processes = {} + def setUp(self): + super(IntegrationTestCase, self).setUp() + self._stop_running_processes() + def tearDown(self): super(IntegrationTestCase, self).tearDown() + self._stop_running_processes() + def _stop_running_processes(self): # Make sure we kill all the processes on teardown so they don't linger around if an # exception was thrown. - for pid, process in self.processes.items(): + pids = list(self.processes.keys()) + for pid in pids: + process = self.processes[pid] try: process.kill() @@ -564,6 +572,8 @@ def tearDown(self): # Process already exited or similar pass + del self.processes[pid] + if self.print_stdout_stderr_on_teardown: try: stdout = process.stdout.read() diff --git a/tox.ini b/tox.ini deleted file mode 100644 index de40b85878..0000000000 --- a/tox.ini +++ /dev/null @@ -1,92 +0,0 @@ -[tox] -envlist = py36-unit,py36-integration -skipsdist = true -skip_missing_interpreters=true - -[testenv] -setenv = PYTHONPATH = {toxinidir}/external - VIRTUALENV_DIR = {envdir} -passenv = NOSE_WITH_TIMER TRAVIS ST2_CI -install_command = pip install -U --force-reinstall {opts} {packages} -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt - -e{toxinidir}/st2tests - -e{toxinidir}/st2actions - -e{toxinidir}/st2api - -e{toxinidir}/st2auth - -e{toxinidir}/st2client - -e{toxinidir}/st2common - -e{toxinidir}/st2reactor - -# Python 3 tasks -[testenv:py36-unit] -basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner:{toxinidir}/st2common/tests/runners/mock_query_callback - VIRTUALENV_DIR = {envdir} -passenv = NOSE_WITH_TIMER TRAVIS ST2_CI -install_command = pip install -U --force-reinstall {opts} {packages} -deps = virtualenv - -r{toxinidir}/requirements.txt - -e{toxinidir}/st2client - -e{toxinidir}/st2common - -e{toxinidir}/st2auth -commands = - nosetests --rednose --immediate -sv st2actions/tests/unit/ - nosetests --rednose --immediate -sv st2auth/tests/unit/ - nosetests --rednose --immediate -sv st2api/tests/unit/controllers/v1/ - nosetests --rednose --immediate -sv st2api/tests/unit/controllers/exp/ - nosetests --rednose --immediate -sv st2common/tests/unit/ - nosetests --rednose --immediate -sv st2client/tests/unit/ - nosetests --rednose --immediate -sv st2exporter/tests/unit/ - nosetests --rednose --immediate -sv st2reactor/tests/unit/ - nosetests --rednose --immediate -sv st2stream/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/action_chain_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/inquirer_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/announcement_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/http_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/noop_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/local_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/orquesta_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/python_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/winrm_runner/tests/unit/ - -[testenv:py36-packs] -basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner - VIRTUALENV_DIR = {envdir} -passenv = NOSE_WITH_TIMER TRAVIS ST2_CI -install_command = pip install -U --force-reinstall {opts} {packages} -deps = virtualenv - -r{toxinidir}/requirements.txt - -e{toxinidir}/st2client - -e{toxinidir}/st2common -commands = - st2-run-pack-tests -c -t -x -p contrib/packs - st2-run-pack-tests -c -t -x -p contrib/core - st2-run-pack-tests -c -t -x -p contrib/default - st2-run-pack-tests -c -t -x -p contrib/chatops - st2-run-pack-tests -c -t -x -p contrib/examples - st2-run-pack-tests -c -t -x -p contrib/linux - st2-run-pack-tests -c -t -x -p contrib/hello_st2 - -[testenv:py36-integration] -basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2auth:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner - VIRTUALENV_DIR = {envdir} -passenv = NOSE_WITH_TIMER TRAVIS ST2_CI -install_command = pip install -U --force-reinstall {opts} {packages} -deps = virtualenv - -r{toxinidir}/requirements.txt - -e{toxinidir}/st2client - -e{toxinidir}/st2common -commands = - nosetests --rednose --immediate -sv st2actions/tests/integration/ - nosetests --rednose --immediate -sv st2api/tests/integration/ - nosetests --rednose --immediate -sv st2common/tests/integration/ - nosetests --rednose --immediate -sv st2exporter/tests/integration/ - nosetests --rednose --immediate -sv st2reactor/tests/integration/ - nosetests --rednose --immediate -sv contrib/runners/action_chain_runner/tests/integration/ - nosetests --rednose --immediate -sv contrib/runners/local_runner/tests/integration/ - nosetests --rednose --immediate -sv contrib/runners/orquesta_runner/tests/integration/ - nosetests --rednose --immediate -sv st2tests/integration/orquesta/ - nosetests --rednose --immediate -sv contrib/runners/python_runner/tests/integration/