From 14b1dc3b75e69ba13bfb3c7f46770938dea15468 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 30 Jul 2019 21:08:02 +0200 Subject: [PATCH 01/15] Fix and combine integration tests targets. Make sure we only generate coverage for runners and orquesta integration tests when ENABLE_COVERAGE environment variable is set to "yes". Coverage adds a lot of overhead so this should speed up PR builds. --- Makefile | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 08fec2c261..b7f4a3780a 100644 --- a/Makefile +++ b/Makefile @@ -665,6 +665,28 @@ endif echo "Done running tests in" $$component; \ echo "==========================================================="; \ done + @echo + @echo "============== runners integration tests with coverage ==============" + @echo + @echo "The tests assume st2 is running on 127.0.0.1." + @for component in $(COMPONENTS_RUNNERS); do\ + echo "==========================================================="; \ + echo "Running tests in" $$component; \ + echo "==========================================================="; \ + . $(VIRTUALENV_DIR)/bin/activate; \ + COVERAGE_FILE=.coverage.integration.$$(echo $$component | tr '/' '.') \ + nosetests $(NOSE_OPTS) -s -v \ + $(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) $$component/tests/integration || exit 1; \ + done + @echo + @echo "==================== Orquesta integration tests with coverage (HTML reports) ====================" + @echo "The tests assume st2 is running on 127.0.0.1." + @echo + . $(VIRTUALENV_DIR)/bin/activate; \ + COVERAGE_FILE=.coverage.integration.orquesta \ + nosetests $(NOSE_OPTS) -s -v \ + $(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) st2tests/integration/orquesta || exit 1; \ + .PHONY: .combine-integration-tests-coverage .combine-integration-tests-coverage: .run-integration-tests-coverage @@ -960,7 +982,7 @@ ci-unit: .unit-tests-coverage-html sudo -E ./scripts/travis/prepare-integration.sh .PHONY: ci-integration -ci-integration: .ci-prepare-integration .itests-coverage-html .runners-itests-coverage-html .orquesta-itests-coverage-html +ci-integration: .ci-prepare-integration .itests-coverage-html .PHONY: ci-runners ci-runners: .ci-prepare-integration .runners-itests-coverage-html From 2c1d1a5462f3a492155f9d78fa7c31417b13a5ea Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 30 Jul 2019 22:22:19 +0200 Subject: [PATCH 02/15] Try to decrease wait delay, see if that helps. --- st2tests/integration/orquesta/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2tests/integration/orquesta/base.py b/st2tests/integration/orquesta/base.py index dcb4206260..697a700b1c 100644 --- a/st2tests/integration/orquesta/base.py +++ b/st2tests/integration/orquesta/base.py @@ -84,7 +84,7 @@ def _execute_workflow(self, action, parameters=None, execute_async=True, @retrying.retry( retry_on_exception=retry_on_exceptions, - wait_fixed=3000, stop_max_delay=900000) + wait_fixed=500, stop_max_delay=900000) def _wait_for_state(self, ex, states): if isinstance(states, six.string_types): states = [states] @@ -113,7 +113,7 @@ def _get_children(self, ex): @retrying.retry( retry_on_exception=retry_on_exceptions, - wait_fixed=3000, stop_max_delay=900000) + wait_fixed=500, stop_max_delay=900000) def _wait_for_task(self, ex, task, status=None, num_task_exs=1): ex = self.st2client.executions.get_by_id(ex.id) From 8fc1423056daa486e37c95e1ca48c38367569869 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 31 Jul 2019 12:06:18 +0200 Subject: [PATCH 03/15] Add a work around for two tests which rely on longer retry delay. --- .../integration/orquesta/test_wiring_error_handling.py | 7 +++++++ .../integration/orquesta/test_wiring_pause_and_resume.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/st2tests/integration/orquesta/test_wiring_error_handling.py b/st2tests/integration/orquesta/test_wiring_error_handling.py index 7f70dad869..be49756b31 100644 --- a/st2tests/integration/orquesta/test_wiring_error_handling.py +++ b/st2tests/integration/orquesta/test_wiring_error_handling.py @@ -14,6 +14,7 @@ from __future__ import absolute_import +import eventlet from integration.orquesta import base from st2common.constants import action as ac_const @@ -244,6 +245,12 @@ def test_remediate_then_fail(self): ex = self._wait_for_completion(ex) # Assert that the log task is executed. + # NOTE: There is a race wheen execution gets in a desired state, but before the child + # tasks are written. To avoid that, we use longer sleep delay here. + # Better approach would be to try to retry a couple of times until expected num of + # tasks is reached (With some hard limit) before failing + eventlet.sleep(2) + self._wait_for_task(ex, 'task1', ac_const.LIVEACTION_STATUS_FAILED) self._wait_for_task(ex, 'log', ac_const.LIVEACTION_STATUS_SUCCEEDED) diff --git a/st2tests/integration/orquesta/test_wiring_pause_and_resume.py b/st2tests/integration/orquesta/test_wiring_pause_and_resume.py index 6e8a3abe7e..a2bd3ec69f 100644 --- a/st2tests/integration/orquesta/test_wiring_pause_and_resume.py +++ b/st2tests/integration/orquesta/test_wiring_pause_and_resume.py @@ -15,6 +15,7 @@ from __future__ import absolute_import import os +import eventlet from integration.orquesta import base @@ -177,6 +178,12 @@ def test_pause_and_resume_cascade_from_subworkflow(self): self.assertFalse(os.path.exists(path)) # Wait for the workflow and task to be paused. + # NOTE: There is a race wheen execution gets in a desired state, but before the child + # tasks are written. To avoid that, we use longer sleep delay here. + # Better approach would be to try to retry a couple of times until expected num of + # tasks is reached (With some hard limit) before failing + eventlet.sleep(3) + tk_ac_ex = self._wait_for_state(tk_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED) ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_PAUSED) From bbebd0284eba2e76ef3d3f255920554cb12b349a Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 31 Jul 2019 13:20:07 +0200 Subject: [PATCH 04/15] Allow wait_fixed and stop_max_delay to be provided on per method invocation basis. Update tests which have a race / rely on timing to use longer wait time to avoid failure. --- st2tests/integration/orquesta/base.py | 148 ++++++++++-------- .../orquesta/test_wiring_pause_and_resume.py | 76 +++++---- 2 files changed, 127 insertions(+), 97 deletions(-) diff --git a/st2tests/integration/orquesta/base.py b/st2tests/integration/orquesta/base.py index 697a700b1c..b7076fd4ce 100644 --- a/st2tests/integration/orquesta/base.py +++ b/st2tests/integration/orquesta/base.py @@ -32,6 +32,9 @@ action_constants.LIVEACTION_STATUS_RUNNING ] +DEFAULT_WAIT_FIXED = 3000 +DEFAULT_STOP_MAX_DELAY = 900000 + def retry_on_exceptions(exc): return isinstance(exc, AssertionError) @@ -82,86 +85,99 @@ def _execute_workflow(self, action, parameters=None, execute_async=True, return ex - @retrying.retry( - retry_on_exception=retry_on_exceptions, - wait_fixed=500, stop_max_delay=900000) - def _wait_for_state(self, ex, states): - if isinstance(states, six.string_types): - states = [states] + def _wait_for_state(self, ex, states, wait_fixed=DEFAULT_WAIT_FIXED, + stop_max_delay=DEFAULT_STOP_MAX_DELAY): - for state in states: - if state not in action_constants.LIVEACTION_STATUSES: - raise ValueError('Status %s is not valid.' % state) + @retrying.retry( + retry_on_exception=retry_on_exceptions, + wait_fixed=wait_fixed, stop_max_delay=stop_max_delay) + def inner(ex, states): + if isinstance(states, six.string_types): + states = [states] - try: - ex = self.st2client.executions.get_by_id(ex.id) - self.assertIn(ex.status, states) - except: - if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: - raise Exception( - 'Execution is in completed state "%s" and ' - 'does not match expected state(s). %s' % - (ex.status, ex.result) - ) - else: - raise + for state in states: + if state not in action_constants.LIVEACTION_STATUSES: + raise ValueError('Status %s is not valid.' % state) - return ex + try: + ex = self.st2client.executions.get_by_id(ex.id) + self.assertIn(ex.status, states) + except: + if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: + raise Exception( + 'Execution is in completed state "%s" and ' + 'does not match expected state(s). %s' % + (ex.status, ex.result) + ) + else: + raise + + return ex + return inner(ex=ex, states=states) def _get_children(self, ex): return self.st2client.executions.query(parent=ex.id) - @retrying.retry( - retry_on_exception=retry_on_exceptions, - wait_fixed=500, stop_max_delay=900000) - def _wait_for_task(self, ex, task, status=None, num_task_exs=1): - ex = self.st2client.executions.get_by_id(ex.id) - - task_exs = [ - task_ex for task_ex in self._get_children(ex) - if task_ex.context.get('orquesta', {}).get('task_name', '') == task - ] - - try: - self.assertEqual(len(task_exs), num_task_exs) - except: - if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: - raise Exception( - 'Execution is in completed state and does not match expected number of ' - 'tasks. Expected: %s Actual: %s' % (str(num_task_exs), str(len(task_exs))) - ) - else: - raise + def _wait_for_task(self, ex, task, status=None, num_task_exs=1, + wait_fixed=DEFAULT_WAIT_FIXED, + stop_max_delay=DEFAULT_STOP_MAX_DELAY): + + @retrying.retry( + retry_on_exception=retry_on_exceptions, + wait_fixed=wait_fixed, stop_max_delay=stop_max_delay) + def inner(ex, task, status, num_task_exs): + ex = self.st2client.executions.get_by_id(ex.id) + + task_exs = [ + task_ex for task_ex in self._get_children(ex) + if task_ex.context.get('orquesta', {}).get('task_name', '') == task + ] - if status is not None: try: - self.assertTrue(all([task_ex.status == status for task_ex in task_exs])) + self.assertEqual(len(task_exs), num_task_exs) except: if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: raise Exception( - 'Execution is in completed state and not all tasks ' - 'match expected status "%s".' % status + 'Execution is in completed state and does not match expected number of ' + 'tasks. Expected: %s Actual: %s' % (str(num_task_exs), str(len(task_exs))) ) else: raise - return task_exs - - @retrying.retry( - retry_on_exception=retry_on_exceptions, - wait_fixed=3000, stop_max_delay=900000) - def _wait_for_completion(self, ex): - ex = self._wait_for_state(ex, action_constants.LIVEACTION_COMPLETED_STATES) - - try: - self.assertTrue(hasattr(ex, 'result')) - except: - if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: - raise Exception( - 'Execution is in completed state and does not ' - 'contain expected result.' - ) - else: - raise + if status is not None: + try: + self.assertTrue(all([task_ex.status == status for task_ex in task_exs])) + except: + if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: + raise Exception( + 'Execution is in completed state and not all tasks ' + 'match expected status "%s".' % status + ) + else: + raise + + return task_exs + return inner(ex=ex, task=task, status=status, num_task_exs=num_task_exs) + + def _wait_for_completion(self, ex, wait_fixed=DEFAULT_WAIT_FIXED, + stop_max_delay=DEFAULT_STOP_MAX_DELAY): + + @retrying.retry( + retry_on_exception=retry_on_exceptions, + wait_fixed=wait_fixed, stop_max_delay=stop_max_delay) + def inner(ex): + ex = self._wait_for_state(ex, action_constants.LIVEACTION_COMPLETED_STATES) - return ex + try: + self.assertTrue(hasattr(ex, 'result')) + except: + if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: + raise Exception( + 'Execution is in completed state and does not ' + 'contain expected result.' + ) + else: + raise + + return ex + return inner(ex=ex) diff --git a/st2tests/integration/orquesta/test_wiring_pause_and_resume.py b/st2tests/integration/orquesta/test_wiring_pause_and_resume.py index a2bd3ec69f..08aaadbc1f 100644 --- a/st2tests/integration/orquesta/test_wiring_pause_and_resume.py +++ b/st2tests/integration/orquesta/test_wiring_pause_and_resume.py @@ -15,7 +15,6 @@ from __future__ import absolute_import import os -import eventlet from integration.orquesta import base @@ -162,38 +161,39 @@ def test_pause_and_resume_cascade_from_subworkflow(self): # Launch the workflow. The workflow will wait for the temp file to be deleted. params = {'tempfile': path} ex = self._execute_workflow('examples.orquesta-test-pause-subworkflow', params) - ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING) - tk_exs = self._wait_for_task(ex, 'task1', ac_const.LIVEACTION_STATUS_RUNNING) + + ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING, wait_fixed=1500) + tk_exs = self._wait_for_task(ex, 'task1', ac_const.LIVEACTION_STATUS_RUNNING, + wait_fixed=1500) # Pause the subworkflow before the temp file is deleted. The task will be # paused but workflow will still be running. tk_ac_ex = self.st2client.executions.pause(tk_exs[0].id) # Expecting the workflow is still running and task1 is pausing. - ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING) - tk_ac_ex = self._wait_for_state(tk_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING) + ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING, + wait_fixed=1500) + tk_ac_ex = self._wait_for_state(tk_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING, + wait_fixed=1500) # Delete the temporary file. os.remove(path) self.assertFalse(os.path.exists(path)) - # Wait for the workflow and task to be paused. - # NOTE: There is a race wheen execution gets in a desired state, but before the child - # tasks are written. To avoid that, we use longer sleep delay here. - # Better approach would be to try to retry a couple of times until expected num of - # tasks is reached (With some hard limit) before failing - eventlet.sleep(3) - - tk_ac_ex = self._wait_for_state(tk_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED) - ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_PAUSED) + tk_ac_ex = self._wait_for_state(tk_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED, + wait_fixed=1500) + ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_PAUSED, + wait_fixed=1500) # Resume the task. tk_ac_ex = self.st2client.executions.resume(tk_ac_ex.id) # Wait for completion. - tk_ac_ex = self._wait_for_state(tk_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED) - ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_SUCCEEDED) - + tk_ac_ex = self._wait_for_state(tk_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED, + wait_fixed=1500) + ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_SUCCEEDED, + wait_fixed=1500) + def test_pause_from_1_of_2_subworkflows_and_resume_subworkflow_when_workflow_paused(self): # Temp files are created during test setup. Ensure the temp files exist. path1 = self.temp_file_path_x @@ -305,8 +305,10 @@ def test_pause_from_all_subworkflows_and_resume_from_subworkflows(self): params = {'file1': path1, 'file2': path2} ex = self._execute_workflow('examples.orquesta-test-pause-subworkflows', params) ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING) - tk1_exs = self._wait_for_task(ex, 'task1', ac_const.LIVEACTION_STATUS_RUNNING) - tk2_exs = self._wait_for_task(ex, 'task2', ac_const.LIVEACTION_STATUS_RUNNING) + tk1_exs = self._wait_for_task(ex, 'task1', ac_const.LIVEACTION_STATUS_RUNNING, + wait_fixed=1500) + tk2_exs = self._wait_for_task(ex, 'task2', ac_const.LIVEACTION_STATUS_RUNNING, + wait_fixed=1500) # Pause the subworkflow before the temp file is deleted. The task will be # paused but workflow and the other subworkflow will still be running. @@ -314,8 +316,10 @@ def test_pause_from_all_subworkflows_and_resume_from_subworkflows(self): # Expecting the workflow is still running and task1 is pausing. ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING) - tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING) - tk2_ac_ex = self._wait_for_state(tk2_exs[0], ac_const.LIVEACTION_STATUS_RUNNING) + tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING, + wait_fixed=1500) + tk2_ac_ex = self._wait_for_state(tk2_exs[0], ac_const.LIVEACTION_STATUS_RUNNING, + wait_fixed=1500) # Pause the other subworkflow before the temp file is deleted. The main # workflow will still running because pause is initiated downstream. @@ -323,8 +327,10 @@ def test_pause_from_all_subworkflows_and_resume_from_subworkflows(self): # Expecting workflow and subworkflows are pausing. ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING) - tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING) - tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING) + tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING, + wait_fixed=1500) + tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING, + wait_fixed=1500) # Delete the temporary files for the subworkflows. os.remove(path1) @@ -334,25 +340,33 @@ def test_pause_from_all_subworkflows_and_resume_from_subworkflows(self): # Wait for subworkflows to pause. The main workflow will also # pause now because no other task is running. - tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED) - tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED) + tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED, + wait_fixed=1500) + tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED, + wait_fixed=1500) ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_PAUSED) # Resume the subworkflow. tk1_ac_ex = self.st2client.executions.resume(tk1_ac_ex.id) # The subworkflow will succeed while the other subworkflow is still paused. - tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED) - tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED) - ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_PAUSED) + tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED, + wait_fixed=1500) + tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED, + wait_fixed=1500) + ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_PAUSED, + wait_fixed=1500) # Resume the other subworkflow. tk2_ac_ex = self.st2client.executions.resume(tk2_ac_ex.id) # Wait for completion. - tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED) - tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED) - ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_SUCCEEDED) + tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED, + wait_fixed=1500) + tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED, + wait_fixed=1500) + ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_SUCCEEDED, + wait_fixed=1500) def test_pause_from_all_subworkflows_and_resume_from_parent_workflow(self): # Temp files are created during test setup. Ensure the temp files exist. From f9cd5488a2b91b9f895311e75db582a4d4230dfb Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 31 Jul 2019 13:42:27 +0200 Subject: [PATCH 05/15] WIP: Move slow mistral unit tests and Orquesta integration tests to a nightly build. This should substantially speed up PR builds. --- .travis.yml | 2 +- Makefile | 18 ++++++++++++++++++ tox.ini | 29 ++++++++++++++++++++++++++--- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e2ccc901fa..1e9e9f980a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -135,7 +135,7 @@ script: # as long as PR builds - if [ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ "${IS_NIGHTLY_BUILD}" = "no" ]; then COMMAND_THRESHOLD=$(expr ${COMMAND_THRESHOLD} \* 2); fi; ./scripts/travis/time-command.sh "make ${TASK}" ${COMMAND_THRESHOLD} # Run any additional nightly checks only as part of a nightly (cron) build - - if [ "${IS_NIGHTLY_BUILD}" = "yes" ] && [ "${TASK}" = "ci-checks ci-packs-tests" ]; then make ci-checks-nightly; fi + - if [ "${IS_NIGHTLY_BUILD}" = "yes" ]; then make ${TASK}-nightly; fi # NOTE: We only generate and submit coverage report for master and version branches # NOTE: We put this here and not after_success so build is marked as failed if this step fails # See https://docs.travis-ci.com/user/customizing-the-build/#breaking-the-build diff --git a/Makefile b/Makefile index b7f4a3780a..870a5c41b4 100644 --- a/Makefile +++ b/Makefile @@ -939,6 +939,13 @@ ci-py3-unit: NOSE_WITH_TIMER=$(NOSE_WITH_TIMER) tox -e py36-unit -vv NOSE_WITH_TIMER=$(NOSE_WITH_TIMER) tox -e py36-packs -vv +.PHONY: ci-py3-unit-nightly +ci-py3-unit: + @echo + @echo "==================== ci-py3-unit ====================" + @echo + NOSE_WITH_TIMER=$(NOSE_WITH_TIMER) tox -e py36-unit-nightly -vv + .PHONY: ci-py3-integration ci-py3-integration: requirements .ci-prepare-integration .ci-py3-integration @@ -949,6 +956,17 @@ ci-py3-integration: requirements .ci-prepare-integration .ci-py3-integration @echo NOSE_WITH_TIMER=$(NOSE_WITH_TIMER) tox -e py36-integration -vv +.PHONY: ci-py3-integration +ci-py3-integration: requirements .ci-prepare-integration .ci-py3-integration-nightly + +.PHONY: .ci-py3-integration-nightly +.ci-py3-integration-nightly: + @echo + @echo "==================== ci-py3-integration ====================" + @echo + NOSE_WITH_TIMER=$(NOSE_WITH_TIMER) tox -e py36-integration-nightly -vv + + .PHONY: .rst-check .rst-check: @echo diff --git a/tox.ini b/tox.ini index fb193b3449..7d6578f4c4 100644 --- a/tox.ini +++ b/tox.ini @@ -46,12 +46,23 @@ commands = nosetests --rednose --immediate -sv contrib/runners/http_runner/tests/unit/ nosetests --rednose --immediate -sv contrib/runners/noop_runner/tests/unit/ nosetests --rednose --immediate -sv contrib/runners/local_runner/tests/unit/ - nosetests --rednose --immediate -sv contrib/runners/mistral_v2/tests/unit/ nosetests --rednose --immediate -sv contrib/runners/orquesta_runner/tests/unit/ nosetests --rednose --immediate -sv contrib/runners/python_runner/tests/unit/ nosetests --rednose --immediate -sv contrib/runners/winrm_runner/tests/unit/ -# Python 3 tasks +[testenv:py36-unit-nightly] +basepython = python3.6 +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner + VIRTUALENV_DIR = {envdir} +passenv = NOSE_WITH_TIMER TRAVIS +install_command = pip install -U --force-reinstall {opts} {packages} +deps = virtualenv + -r{toxinidir}/requirements.txt + -e{toxinidir}/st2client + -e{toxinidir}/st2common +commands = + nosetests --rednose --immediate -sv contrib/runners/mistral_v2/tests/unit/ + [testenv:py36-packs] basepython = python3.6 setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner @@ -91,6 +102,18 @@ commands = nosetests --rednose --immediate -sv --exe contrib/runners/action_chain_runner/tests/integration/ nosetests --rednose --immediate -sv --exe contrib/runners/local_runner/tests/integration/ nosetests --rednose --immediate -sv --exe contrib/runners/mistral_v2/tests/integration/ - nosetests --rednose --immediate -sv --exe contrib/runners/orquesta_runner/tests/integration/ nosetests --rednose --immediate -sv --exe st2tests/integration/orquesta/ nosetests --rednose --immediate -sv --exe contrib/runners/python_runner/tests/integration/ + +[testenv:py36-integration-nightly] +basepython = python3.6 +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2auth:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner + VIRTUALENV_DIR = {envdir} +passenv = NOSE_WITH_TIMER TRAVIS +install_command = pip install -U --force-reinstall {opts} {packages} +deps = virtualenv + -r{toxinidir}/requirements.txt + -e{toxinidir}/st2client + -e{toxinidir}/st2common +commands = + nosetests --rednose --immediate -sv --exe contrib/runners/orquesta_runner/tests/integration/ From 01c20bd65d0800874acc3dace0978db4fae776fc Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 8 Aug 2019 15:45:29 +0200 Subject: [PATCH 06/15] Update Travis config to only run nightly build if a particular task exists. --- .travis.yml | 2 +- scripts/travis/run-make-task-if-exists.sh | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100755 scripts/travis/run-make-task-if-exists.sh diff --git a/.travis.yml b/.travis.yml index 1e9e9f980a..077eab2970 100644 --- a/.travis.yml +++ b/.travis.yml @@ -135,7 +135,7 @@ script: # as long as PR builds - if [ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ "${IS_NIGHTLY_BUILD}" = "no" ]; then COMMAND_THRESHOLD=$(expr ${COMMAND_THRESHOLD} \* 2); fi; ./scripts/travis/time-command.sh "make ${TASK}" ${COMMAND_THRESHOLD} # Run any additional nightly checks only as part of a nightly (cron) build - - if [ "${IS_NIGHTLY_BUILD}" = "yes" ]; then make ${TASK}-nightly; fi + - if [ "${IS_NIGHTLY_BUILD}" = "yes" ]; then ./scripts/travis/run-make-task-if-exists.sh ${TASK}-nightly; fi # NOTE: We only generate and submit coverage report for master and version branches # NOTE: We put this here and not after_success so build is marked as failed if this step fails # See https://docs.travis-ci.com/user/customizing-the-build/#breaking-the-build diff --git a/scripts/travis/run-make-task-if-exists.sh b/scripts/travis/run-make-task-if-exists.sh new file mode 100755 index 0000000000..47544b7e87 --- /dev/null +++ b/scripts/travis/run-make-task-if-exists.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Script which runs make tasks if it exists. If it doesn't exist, it exits with +# 0 status code +TASK=$1 + +if [ ! "${TASK}" ]; then + echo "Missing TASK argument" + exit 2 +fi + +$(make -n ${TASK} &> /dev/null) + +if [ $? -eq 2 ]; then + # tASK DOESN'T EXIST + echo "Task ${TASK} doesn't exist, skipping execution..." + exit 0 +fi + +exec make ${TASK} From 036c680b4d9856eff81a7f8b535847c73c5ceec3 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 8 Aug 2019 15:54:33 +0200 Subject: [PATCH 07/15] Use changes from master. --- st2tests/integration/orquesta/base.py | 147 ++++++++---------- .../orquesta/test_wiring_pause_and_resume.py | 67 +++----- 2 files changed, 90 insertions(+), 124 deletions(-) diff --git a/st2tests/integration/orquesta/base.py b/st2tests/integration/orquesta/base.py index b7076fd4ce..a1f16dbb81 100644 --- a/st2tests/integration/orquesta/base.py +++ b/st2tests/integration/orquesta/base.py @@ -32,7 +32,7 @@ action_constants.LIVEACTION_STATUS_RUNNING ] -DEFAULT_WAIT_FIXED = 3000 +DEFAULT_WAIT_FIXED = 500 DEFAULT_STOP_MAX_DELAY = 900000 @@ -85,99 +85,86 @@ def _execute_workflow(self, action, parameters=None, execute_async=True, return ex - def _wait_for_state(self, ex, states, wait_fixed=DEFAULT_WAIT_FIXED, - stop_max_delay=DEFAULT_STOP_MAX_DELAY): + @retrying.retry( + retry_on_exception=retry_on_exceptions, + wait_fixed=DEFAULT_WAIT_FIXED, stop_max_delay=DEFAULT_STOP_MAX_DELAY) + def _wait_for_state(self, ex, states): + if isinstance(states, six.string_types): + states = [states] - @retrying.retry( - retry_on_exception=retry_on_exceptions, - wait_fixed=wait_fixed, stop_max_delay=stop_max_delay) - def inner(ex, states): - if isinstance(states, six.string_types): - states = [states] + for state in states: + if state not in action_constants.LIVEACTION_STATUSES: + raise ValueError('Status %s is not valid.' % state) - for state in states: - if state not in action_constants.LIVEACTION_STATUSES: - raise ValueError('Status %s is not valid.' % state) - - try: - ex = self.st2client.executions.get_by_id(ex.id) - self.assertIn(ex.status, states) - except: - if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: - raise Exception( - 'Execution is in completed state "%s" and ' - 'does not match expected state(s). %s' % - (ex.status, ex.result) - ) - else: - raise + try: + ex = self.st2client.executions.get_by_id(ex.id) + self.assertIn(ex.status, states) + except: + if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: + raise Exception( + 'Execution is in completed state "%s" and ' + 'does not match expected state(s). %s' % + (ex.status, ex.result) + ) + else: + raise - return ex - return inner(ex=ex, states=states) + return ex def _get_children(self, ex): return self.st2client.executions.query(parent=ex.id) - def _wait_for_task(self, ex, task, status=None, num_task_exs=1, - wait_fixed=DEFAULT_WAIT_FIXED, - stop_max_delay=DEFAULT_STOP_MAX_DELAY): - - @retrying.retry( - retry_on_exception=retry_on_exceptions, - wait_fixed=wait_fixed, stop_max_delay=stop_max_delay) - def inner(ex, task, status, num_task_exs): - ex = self.st2client.executions.get_by_id(ex.id) - - task_exs = [ - task_ex for task_ex in self._get_children(ex) - if task_ex.context.get('orquesta', {}).get('task_name', '') == task - ] + @retrying.retry( + retry_on_exception=retry_on_exceptions, + wait_fixed=DEFAULT_WAIT_FIXED, stop_max_delay=DEFAULT_STOP_MAX_DELAY) + def _wait_for_task(self, ex, task, status=None, num_task_exs=1): + ex = self.st2client.executions.get_by_id(ex.id) + + task_exs = [ + task_ex for task_ex in self._get_children(ex) + if task_ex.context.get('orquesta', {}).get('task_name', '') == task + ] + + try: + self.assertEqual(len(task_exs), num_task_exs) + except: + if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: + raise Exception( + 'Execution is in completed state and does not match expected number of ' + 'tasks. Expected: %s Actual: %s' % (str(num_task_exs), str(len(task_exs))) + ) + else: + raise + if status is not None: try: - self.assertEqual(len(task_exs), num_task_exs) + self.assertTrue(all([task_ex.status == status for task_ex in task_exs])) except: if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: raise Exception( - 'Execution is in completed state and does not match expected number of ' - 'tasks. Expected: %s Actual: %s' % (str(num_task_exs), str(len(task_exs))) + 'Execution is in completed state and not all tasks ' + 'match expected status "%s".' % status ) else: raise - if status is not None: - try: - self.assertTrue(all([task_ex.status == status for task_ex in task_exs])) - except: - if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: - raise Exception( - 'Execution is in completed state and not all tasks ' - 'match expected status "%s".' % status - ) - else: - raise - - return task_exs - return inner(ex=ex, task=task, status=status, num_task_exs=num_task_exs) - - def _wait_for_completion(self, ex, wait_fixed=DEFAULT_WAIT_FIXED, - stop_max_delay=DEFAULT_STOP_MAX_DELAY): - - @retrying.retry( - retry_on_exception=retry_on_exceptions, - wait_fixed=wait_fixed, stop_max_delay=stop_max_delay) - def inner(ex): - ex = self._wait_for_state(ex, action_constants.LIVEACTION_COMPLETED_STATES) - - try: - self.assertTrue(hasattr(ex, 'result')) - except: - if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: - raise Exception( - 'Execution is in completed state and does not ' - 'contain expected result.' - ) - else: - raise + return task_exs + + @retrying.retry( + retry_on_exception=retry_on_exceptions, + wait_fixed=DEFAULT_WAIT_FIXED, stop_max_delay=DEFAULT_STOP_MAX_DELAY) + def _wait_for_completion(self, ex): + ex = self._wait_for_state(ex, action_constants.LIVEACTION_COMPLETED_STATES) + + try: + self.assertTrue(hasattr(ex, 'result')) + except: + if ex.status in action_constants.LIVEACTION_COMPLETED_STATES: + raise Exception( + 'Execution is in completed state and does not ' + 'contain expected result.' + ) + else: + raise - return ex - return inner(ex=ex) + return ex diff --git a/st2tests/integration/orquesta/test_wiring_pause_and_resume.py b/st2tests/integration/orquesta/test_wiring_pause_and_resume.py index e0f65d495c..6e8a3abe7e 100644 --- a/st2tests/integration/orquesta/test_wiring_pause_and_resume.py +++ b/st2tests/integration/orquesta/test_wiring_pause_and_resume.py @@ -161,38 +161,31 @@ def test_pause_and_resume_cascade_from_subworkflow(self): # Launch the workflow. The workflow will wait for the temp file to be deleted. params = {'tempfile': path} ex = self._execute_workflow('examples.orquesta-test-pause-subworkflow', params) - - ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING, wait_fixed=1500) - tk_exs = self._wait_for_task(ex, 'task1', ac_const.LIVEACTION_STATUS_RUNNING, - wait_fixed=1500) + ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING) + tk_exs = self._wait_for_task(ex, 'task1', ac_const.LIVEACTION_STATUS_RUNNING) # Pause the subworkflow before the temp file is deleted. The task will be # paused but workflow will still be running. tk_ac_ex = self.st2client.executions.pause(tk_exs[0].id) # Expecting the workflow is still running and task1 is pausing. - ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING, - wait_fixed=1500) - tk_ac_ex = self._wait_for_state(tk_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING, - wait_fixed=1500) + ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING) + tk_ac_ex = self._wait_for_state(tk_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING) # Delete the temporary file. os.remove(path) self.assertFalse(os.path.exists(path)) - tk_ac_ex = self._wait_for_state(tk_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED, - wait_fixed=1500) - ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_PAUSED, - wait_fixed=1500) + # Wait for the workflow and task to be paused. + tk_ac_ex = self._wait_for_state(tk_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED) + ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_PAUSED) # Resume the task. tk_ac_ex = self.st2client.executions.resume(tk_ac_ex.id) # Wait for completion. - tk_ac_ex = self._wait_for_state(tk_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED, - wait_fixed=1500) - ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_SUCCEEDED, - wait_fixed=1500) + tk_ac_ex = self._wait_for_state(tk_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED) + ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_SUCCEEDED) def test_pause_from_1_of_2_subworkflows_and_resume_subworkflow_when_workflow_paused(self): # Temp files are created during test setup. Ensure the temp files exist. @@ -305,10 +298,8 @@ def test_pause_from_all_subworkflows_and_resume_from_subworkflows(self): params = {'file1': path1, 'file2': path2} ex = self._execute_workflow('examples.orquesta-test-pause-subworkflows', params) ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING) - tk1_exs = self._wait_for_task(ex, 'task1', ac_const.LIVEACTION_STATUS_RUNNING, - wait_fixed=1500) - tk2_exs = self._wait_for_task(ex, 'task2', ac_const.LIVEACTION_STATUS_RUNNING, - wait_fixed=1500) + tk1_exs = self._wait_for_task(ex, 'task1', ac_const.LIVEACTION_STATUS_RUNNING) + tk2_exs = self._wait_for_task(ex, 'task2', ac_const.LIVEACTION_STATUS_RUNNING) # Pause the subworkflow before the temp file is deleted. The task will be # paused but workflow and the other subworkflow will still be running. @@ -316,10 +307,8 @@ def test_pause_from_all_subworkflows_and_resume_from_subworkflows(self): # Expecting the workflow is still running and task1 is pausing. ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING) - tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING, - wait_fixed=1500) - tk2_ac_ex = self._wait_for_state(tk2_exs[0], ac_const.LIVEACTION_STATUS_RUNNING, - wait_fixed=1500) + tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING) + tk2_ac_ex = self._wait_for_state(tk2_exs[0], ac_const.LIVEACTION_STATUS_RUNNING) # Pause the other subworkflow before the temp file is deleted. The main # workflow will still running because pause is initiated downstream. @@ -327,10 +316,8 @@ def test_pause_from_all_subworkflows_and_resume_from_subworkflows(self): # Expecting workflow and subworkflows are pausing. ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_RUNNING) - tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING, - wait_fixed=1500) - tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING, - wait_fixed=1500) + tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING) + tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_PAUSING) # Delete the temporary files for the subworkflows. os.remove(path1) @@ -340,33 +327,25 @@ def test_pause_from_all_subworkflows_and_resume_from_subworkflows(self): # Wait for subworkflows to pause. The main workflow will also # pause now because no other task is running. - tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED, - wait_fixed=1500) - tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED, - wait_fixed=1500) + tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED) + tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED) ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_PAUSED) # Resume the subworkflow. tk1_ac_ex = self.st2client.executions.resume(tk1_ac_ex.id) # The subworkflow will succeed while the other subworkflow is still paused. - tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED, - wait_fixed=1500) - tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED, - wait_fixed=1500) - ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_PAUSED, - wait_fixed=1500) + tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED) + tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_PAUSED) + ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_PAUSED) # Resume the other subworkflow. tk2_ac_ex = self.st2client.executions.resume(tk2_ac_ex.id) # Wait for completion. - tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED, - wait_fixed=1500) - tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED, - wait_fixed=1500) - ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_SUCCEEDED, - wait_fixed=1500) + tk1_ac_ex = self._wait_for_state(tk1_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED) + tk2_ac_ex = self._wait_for_state(tk2_ac_ex, ac_const.LIVEACTION_STATUS_SUCCEEDED) + ex = self._wait_for_state(ex, ac_const.LIVEACTION_STATUS_SUCCEEDED) def test_pause_from_all_subworkflows_and_resume_from_parent_workflow(self): # Temp files are created during test setup. Ensure the temp files exist. From 06f93a41ebcc53da356efa7404610d969b2f1b84 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 8 Aug 2019 15:59:11 +0200 Subject: [PATCH 08/15] Orquesta integration tests have been optimized, no need to run them as part of the nightly builds now. --- tox.ini | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tox.ini b/tox.ini index 7d6578f4c4..1ebef4b8c3 100644 --- a/tox.ini +++ b/tox.ini @@ -104,16 +104,3 @@ commands = nosetests --rednose --immediate -sv --exe contrib/runners/mistral_v2/tests/integration/ nosetests --rednose --immediate -sv --exe st2tests/integration/orquesta/ nosetests --rednose --immediate -sv --exe contrib/runners/python_runner/tests/integration/ - -[testenv:py36-integration-nightly] -basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2auth:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/orquesta_runner:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/winrm_runner - VIRTUALENV_DIR = {envdir} -passenv = NOSE_WITH_TIMER TRAVIS -install_command = pip install -U --force-reinstall {opts} {packages} -deps = virtualenv - -r{toxinidir}/requirements.txt - -e{toxinidir}/st2client - -e{toxinidir}/st2common -commands = - nosetests --rednose --immediate -sv --exe contrib/runners/orquesta_runner/tests/integration/ From 40cffe9c40efa0dad2232b0a6ac6aa9942932c33 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 8 Aug 2019 16:09:14 +0200 Subject: [PATCH 09/15] Update script so it knows how to handle scenario where multiple tasks are specified, but not all of them have corresponding nightly tasks. For example: TASK="foo1 foo2 foo3" and only "foo2-nightly" task exists. In this case, only "foo2-nightly" would run and other would be ignored. --- .travis.yml | 2 +- scripts/travis/run-make-task-if-exists.sh | 20 ---------- .../travis/run-nightly-make-task-if-exists.sh | 38 +++++++++++++++++++ 3 files changed, 39 insertions(+), 21 deletions(-) delete mode 100755 scripts/travis/run-make-task-if-exists.sh create mode 100755 scripts/travis/run-nightly-make-task-if-exists.sh diff --git a/.travis.yml b/.travis.yml index 077eab2970..f8c138407a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -135,7 +135,7 @@ script: # as long as PR builds - if [ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ "${IS_NIGHTLY_BUILD}" = "no" ]; then COMMAND_THRESHOLD=$(expr ${COMMAND_THRESHOLD} \* 2); fi; ./scripts/travis/time-command.sh "make ${TASK}" ${COMMAND_THRESHOLD} # Run any additional nightly checks only as part of a nightly (cron) build - - if [ "${IS_NIGHTLY_BUILD}" = "yes" ]; then ./scripts/travis/run-make-task-if-exists.sh ${TASK}-nightly; fi + - if [ "${IS_NIGHTLY_BUILD}" = "yes" ]; then ./scripts/travis/run-nightly-make-task-if-exists.sh "${TASK}"; fi # NOTE: We only generate and submit coverage report for master and version branches # NOTE: We put this here and not after_success so build is marked as failed if this step fails # See https://docs.travis-ci.com/user/customizing-the-build/#breaking-the-build diff --git a/scripts/travis/run-make-task-if-exists.sh b/scripts/travis/run-make-task-if-exists.sh deleted file mode 100755 index 47544b7e87..0000000000 --- a/scripts/travis/run-make-task-if-exists.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# Script which runs make tasks if it exists. If it doesn't exist, it exits with -# 0 status code -TASK=$1 - -if [ ! "${TASK}" ]; then - echo "Missing TASK argument" - exit 2 -fi - -$(make -n ${TASK} &> /dev/null) - -if [ $? -eq 2 ]; then - # tASK DOESN'T EXIST - echo "Task ${TASK} doesn't exist, skipping execution..." - exit 0 -fi - -exec make ${TASK} diff --git a/scripts/travis/run-nightly-make-task-if-exists.sh b/scripts/travis/run-nightly-make-task-if-exists.sh new file mode 100755 index 0000000000..35e0ad9f21 --- /dev/null +++ b/scripts/travis/run-nightly-make-task-if-exists.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Script which runs a corresponding make nightly tasks if it exists. If a task corresponding +# nightly task doesn't exist, it's ignored. +# +# For example, let's say we have TASK="ci-checks ci-unit ci-pack-tests" and only +# "ci-checks-nightly" make task exists. +# In this scenario, only "ci-check-nightly" tasks would run and other would be ignored. + +TASK=$1 + +if [ ! "${TASK}" ]; then + echo "Missing TASK argument" + echo "Usage: $0 " + exit 2 +fi + +# Note: TASK could contain a list of multiple tasks +TASKS=($TASK) + +EXISTING_TASKS=() +for TASK_NAME in ${TASKS[@]}; do + $(make -n ${TASK_NAME}-nightly &> /dev/null) + + if [ $? -eq 0 ]; then + # Task {TASK}-nightly exists + EXISTING_TASKS+=("$TASK_NAME-nightly") + fi +done + +# Run only tasks which exist +if [ ${#EXISTING_TASKS[@]} -eq 0 ]; then + echo "No existing nightly tasks found..." + exit 0 +fi + +echo "Running the following nightly tasks: ${EXISTING_TASKS[@]}" +exec make ${EXISTING_TASKS[@]} From 6a324b41bd7221f3a0c54b3d1936fa94d8668d7e Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 8 Aug 2019 16:15:11 +0200 Subject: [PATCH 10/15] Fix typo. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 870a5c41b4..f578486f87 100644 --- a/Makefile +++ b/Makefile @@ -940,7 +940,7 @@ ci-py3-unit: NOSE_WITH_TIMER=$(NOSE_WITH_TIMER) tox -e py36-packs -vv .PHONY: ci-py3-unit-nightly -ci-py3-unit: +ci-py3-unit-nightly: @echo @echo "==================== ci-py3-unit ====================" @echo From dc388e23d6df12663db7db530487111a06e2b0ba Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 8 Aug 2019 16:31:08 +0200 Subject: [PATCH 11/15] Move Mistral tests to a nightly build. --- Makefile | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index f578486f87..2c4bedfd91 100644 --- a/Makefile +++ b/Makefile @@ -22,9 +22,11 @@ BINARIES := bin # All components are prefixed by st2 and not .egg-info. COMPONENTS := $(shell ls -a | grep ^st2 | grep -v .egg-info) COMPONENTS_RUNNERS := $(wildcard contrib/runners/*) +COMPONENTS_RUNNERS := $(wildcard contrib/runners/*) COMPONENTS_WITHOUT_ST2TESTS := $(shell ls -a | grep ^st2 | grep -v .egg-info | grep -v st2tests | grep -v st2exporter) COMPONENTS_WITH_RUNNERS := $(COMPONENTS) $(COMPONENTS_RUNNERS) +COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER := $(foreach component,$(filter-out contrib/runners/mistral_v2,$(COMPONENTS_WITH_RUNNERS)),$(component)) COMPONENTS_TEST_DIRS := $(wildcard st2*/tests) $(wildcard contrib/runners/*/tests) @@ -42,6 +44,7 @@ space_char := space_char += COMPONENT_PYTHONPATH = $(subst $(space_char),:,$(realpath $(COMPONENTS_WITH_RUNNERS))) COMPONENTS_TEST := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS_WITH_RUNNERS)),$(component)) +COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER)),$(component)) COMPONENTS_TEST_COMMA := $(subst $(slash),$(dot),$(subst $(space_char),$(comma),$(COMPONENTS_TEST))) COMPONENTS_TEST_MODULES := $(subst $(slash),$(dot),$(COMPONENTS_TEST_DIRS)) COMPONENTS_TEST_MODULES_COMMA := $(subst $(space_char),$(comma),$(COMPONENTS_TEST_MODULES)) @@ -90,6 +93,9 @@ endif .PHONY: all all: requirements configgen check tests +.PHONY: foo +all: requirements configgen check tests + .PHONY: .coverage_globs .coverage_globs: @for coverage_result in $$( \ @@ -109,6 +115,8 @@ play: @echo @echo COMPONENTS_WITH_RUNNERS=$(COMPONENTS_WITH_RUNNERS) @echo + @echo COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER=$(COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER) + @echo @echo COMPONENTS_TEST=$(COMPONENTS_TEST) @echo @echo COMPONENTS_TEST_COMMA=$(COMPONENTS_TEST_COMMA) @@ -119,6 +127,8 @@ play: @echo @echo COMPONENTS_TEST_MODULES_COMMA=$(COMPONENTS_TEST_MODULES_COMMA) @echo + @echo COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER=$(COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER) + @echo @echo COMPONENT_PYTHONPATH=$(COMPONENT_PYTHONPATH) @echo @echo TRAVIS_PULL_REQUEST=$(TRAVIS_PULL_REQUEST) @@ -202,6 +212,15 @@ check-python-packages-nightly: .PHONY: ci-checks-nightly ci-checks-nightly: check-python-packages-nightly +.PHONY: foo1-nightly +foo1-nightly: + echo "foo1" + +.PHONY: foo2-nightly +foo2-nightly: + echo "foo2" + exit 3 + .PHONY: checklogs checklogs: @echo @@ -573,7 +592,7 @@ endif @echo @echo "----- Dropping st2-test db -----" @mongo st2-test --eval "db.dropDatabase();" - for component in $(COMPONENTS_TEST); do\ + for component in $(COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER); do\ echo "==========================================================="; \ echo "Running tests in" $$component; \ echo "-----------------------------------------------------------"; \ @@ -947,7 +966,7 @@ ci-py3-unit-nightly: NOSE_WITH_TIMER=$(NOSE_WITH_TIMER) tox -e py36-unit-nightly -vv .PHONY: ci-py3-integration -ci-py3-integration: requirements .ci-prepare-integration .ci-py3-integration +ci-py3-integration: requirementci-unit-nightlyi-py3-integration .PHONY: .ci-py3-integration .ci-py3-integration: @@ -959,14 +978,6 @@ ci-py3-integration: requirements .ci-prepare-integration .ci-py3-integration .PHONY: ci-py3-integration ci-py3-integration: requirements .ci-prepare-integration .ci-py3-integration-nightly -.PHONY: .ci-py3-integration-nightly -.ci-py3-integration-nightly: - @echo - @echo "==================== ci-py3-integration ====================" - @echo - NOSE_WITH_TIMER=$(NOSE_WITH_TIMER) tox -e py36-integration-nightly -vv - - .PHONY: .rst-check .rst-check: @echo @@ -995,6 +1006,15 @@ ci-py3-integration: requirements .ci-prepare-integration .ci-py3-integration-nig .PHONY: ci-unit ci-unit: .unit-tests-coverage-html +.PHONY: ci-unit-nightly +ci-unit-nightly: + # NOTE: We run mistral runner checks only as part of a nightly build to speed up + # non nightly builds (Mistral will be deprecated in the future) + @echo + @echo "============== ci-unit-nightly ==============" + @echo + nosetests $(NOSE_OPTS) -s -v contrib/runners/mistral_v2/tests/unit + .PHONY: .ci-prepare-integration .ci-prepare-integration: sudo -E ./scripts/travis/prepare-integration.sh From 50221393369e50b58039fbd3578349453f45ee59 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 8 Aug 2019 16:35:39 +0200 Subject: [PATCH 12/15] Add this line back. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 1ebef4b8c3..3832b40c02 100644 --- a/tox.ini +++ b/tox.ini @@ -102,5 +102,6 @@ commands = nosetests --rednose --immediate -sv --exe contrib/runners/action_chain_runner/tests/integration/ nosetests --rednose --immediate -sv --exe contrib/runners/local_runner/tests/integration/ nosetests --rednose --immediate -sv --exe contrib/runners/mistral_v2/tests/integration/ + nosetests --rednose --immediate -sv --exe contrib/runners/orquesta_runner/tests/integration/ nosetests --rednose --immediate -sv --exe st2tests/integration/orquesta/ nosetests --rednose --immediate -sv --exe contrib/runners/python_runner/tests/integration/ From 6aeaa0a02e421ac4be27810f69e7356ca7d175ce Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 8 Aug 2019 16:36:09 +0200 Subject: [PATCH 13/15] Remove dummy / test tasks. --- Makefile | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 2c4bedfd91..bc53018418 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,6 @@ BINARIES := bin # All components are prefixed by st2 and not .egg-info. COMPONENTS := $(shell ls -a | grep ^st2 | grep -v .egg-info) COMPONENTS_RUNNERS := $(wildcard contrib/runners/*) -COMPONENTS_RUNNERS := $(wildcard contrib/runners/*) COMPONENTS_WITHOUT_ST2TESTS := $(shell ls -a | grep ^st2 | grep -v .egg-info | grep -v st2tests | grep -v st2exporter) COMPONENTS_WITH_RUNNERS := $(COMPONENTS) $(COMPONENTS_RUNNERS) @@ -93,9 +92,6 @@ endif .PHONY: all all: requirements configgen check tests -.PHONY: foo -all: requirements configgen check tests - .PHONY: .coverage_globs .coverage_globs: @for coverage_result in $$( \ @@ -212,15 +208,6 @@ check-python-packages-nightly: .PHONY: ci-checks-nightly ci-checks-nightly: check-python-packages-nightly -.PHONY: foo1-nightly -foo1-nightly: - echo "foo1" - -.PHONY: foo2-nightly -foo2-nightly: - echo "foo2" - exit 3 - .PHONY: checklogs checklogs: @echo @@ -966,7 +953,7 @@ ci-py3-unit-nightly: NOSE_WITH_TIMER=$(NOSE_WITH_TIMER) tox -e py36-unit-nightly -vv .PHONY: ci-py3-integration -ci-py3-integration: requirementci-unit-nightlyi-py3-integration +ci-py3-integration: requirements .ci-prepare-integration .ci-py3-integration .PHONY: .ci-py3-integration .ci-py3-integration: @@ -975,9 +962,6 @@ ci-py3-integration: requirementci-unit-nightlyi-py3-integration @echo NOSE_WITH_TIMER=$(NOSE_WITH_TIMER) tox -e py36-integration -vv -.PHONY: ci-py3-integration -ci-py3-integration: requirements .ci-prepare-integration .ci-py3-integration-nightly - .PHONY: .rst-check .rst-check: @echo From 4347c7bb79da86d29d6c7c4b8bc584ce5c138401 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 8 Aug 2019 16:46:45 +0200 Subject: [PATCH 14/15] Enable Slack notifications. Also enable it so we can test it's working. --- .travis.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.travis.yml b/.travis.yml index f8c138407a..148a7f2734 100644 --- a/.travis.yml +++ b/.travis.yml @@ -147,3 +147,15 @@ script: # Alternative: use strict pip pinning, including git-based pip packages before_cache: - if [ ${TRAVIS_PULL_REQUEST} = 'false' ] && [ "${IS_NIGHTLY_BUILD}" = "no" ]; then rm -rf virtualenv/; fi + +# We want to be notified when a master or nightly build fails +notifications: + # Post build failures to '#stackstorm' channel in 'stackstorm' Slack + slack: + rooms: + - secure: "rPA22aDgvNe0/S/2e+cp1rSDdDUPufLXnCbfnRzMPVDSQ2UPdLmEl9IeOoEHZmq92AZtzY8UnQaPFuoM0HAPrYDgKopn4n4KpOo+xUlJ92qdNj5qk3Z1TmQHwUYFvCkMvaR/CpX2liRr/YB3qM+1vFAMsYgmqrBX8vcEqNJQy/M=" + on_pull_requests: true + on_success: always + on_failure: always + #on_success: change # default: always + #on_failure: always # default: always From ea2971f478d61ebe19b67dfed57151357a057dec Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 8 Aug 2019 17:09:44 +0200 Subject: [PATCH 15/15] Remove test change. --- .travis.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 148a7f2734..60617b9dc1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -154,8 +154,6 @@ notifications: slack: rooms: - secure: "rPA22aDgvNe0/S/2e+cp1rSDdDUPufLXnCbfnRzMPVDSQ2UPdLmEl9IeOoEHZmq92AZtzY8UnQaPFuoM0HAPrYDgKopn4n4KpOo+xUlJ92qdNj5qk3Z1TmQHwUYFvCkMvaR/CpX2liRr/YB3qM+1vFAMsYgmqrBX8vcEqNJQy/M=" - on_pull_requests: true - on_success: always - on_failure: always - #on_success: change # default: always - #on_failure: always # default: always + on_pull_requests: false + on_success: change # default: always + on_failure: always # default: always