From 6c87a97fd7fbca96aa5fc6309ce25a63e303616e Mon Sep 17 00:00:00 2001 From: W Chan Date: Mon, 24 Feb 2020 22:08:49 +0000 Subject: [PATCH 1/3] Rollback field changes on workflow and task execution collection Rollback fields from DictField to EscapedDictField because in various use cases, result may contain dictionary which IP address as key. DictField does not allow "." in the key name. The execution result may be published into the context and into workflow output and then passed as input into subworkflow. Appropriate fields are rolled back to EscapedDictField to avoid breaking existing user workflows. --- .../actions/orquesta-mock-create-vm.yaml | 6 ++++++ .../examples/actions/python-mock-create-vm.py | 21 +++++++++++++++++++ .../actions/python-mock-create-vm.yaml | 19 +++++++++++++++++ .../workflows/orquesta-mock-create-vm.yaml | 16 ++++++++++---- st2common/st2common/models/db/workflow.py | 10 ++++----- .../integration/orquesta/test_performance.py | 2 +- 6 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 contrib/examples/actions/python-mock-create-vm.py create mode 100644 contrib/examples/actions/python-mock-create-vm.yaml diff --git a/contrib/examples/actions/orquesta-mock-create-vm.yaml b/contrib/examples/actions/orquesta-mock-create-vm.yaml index 848b97dd4c..85e774a702 100644 --- a/contrib/examples/actions/orquesta-mock-create-vm.yaml +++ b/contrib/examples/actions/orquesta-mock-create-vm.yaml @@ -14,3 +14,9 @@ parameters: vm_name: required: true type: string + ip: + default: "10.1.23.99" + required: true + type: string + meta: + type: object diff --git a/contrib/examples/actions/python-mock-create-vm.py b/contrib/examples/actions/python-mock-create-vm.py new file mode 100644 index 0000000000..60a88b7967 --- /dev/null +++ b/contrib/examples/actions/python-mock-create-vm.py @@ -0,0 +1,21 @@ +import eventlet +import random + +from st2common.runners.base_action import Action + + +class MockCreateVMAction(Action): + + def run(self, cpu_cores, memory_mb, vm_name, ip): + eventlet.sleep(5) + + data = { + 'vm_id': 'vm' + str(random.randint(0, 10000)), + ip: { + 'cpu_cores': cpu_cores, + 'memory_mb': memory_mb, + 'vm_name': vm_name + } + } + + return data diff --git a/contrib/examples/actions/python-mock-create-vm.yaml b/contrib/examples/actions/python-mock-create-vm.yaml new file mode 100644 index 0000000000..31cfd5d457 --- /dev/null +++ b/contrib/examples/actions/python-mock-create-vm.yaml @@ -0,0 +1,19 @@ +--- +name: python-mock-create-vm +runner_type: python-script +description: Action which mocks creation of a VM. +enabled: true +entry_point: python-mock-create-vm.py +parameters: + cpu_cores: + default: 1 + type: integer + memory_mb: + default: 1024 + type: integer + vm_name: + required: true + type: string + ip: + required: true + type: string diff --git a/contrib/examples/actions/workflows/orquesta-mock-create-vm.yaml b/contrib/examples/actions/workflows/orquesta-mock-create-vm.yaml index 089314217b..f1902c5e17 100644 --- a/contrib/examples/actions/workflows/orquesta-mock-create-vm.yaml +++ b/contrib/examples/actions/workflows/orquesta-mock-create-vm.yaml @@ -6,10 +6,16 @@ input: - vm_name - cpu_cores - memory_mb + - ip + - meta + +vars: + - extra: <% ctx().meta %> output: - vm_id: <% ctx().vm_id %> - ip: <% ctx().ip %> + - extra: <% ctx().extra %> tasks: register_dns: @@ -19,7 +25,7 @@ tasks: next: - when: <% succeeded() %> publish: - - ip: "10.1.23.99" + - ip: <% ctx().ip %> - status_message: "DNS for <% ctx().vm_name %> is registered." do: - configure_vm @@ -27,14 +33,16 @@ tasks: create_vm: - action: core.local + action: examples.python-mock-create-vm input: - cmd: "printf 'vm1234'; sleep 6" + vm_name: <% ctx().vm_name %> + ip: <% ctx().ip %> next: - when: <% succeeded() %> publish: - - vm_id: <% result().stdout %> + - vm_id: <% result().result.vm_id %> - status_message: "VM <% ctx().vm_name %> is created." + - extra: <% result().result %> do: - configure_vm - notify diff --git a/st2common/st2common/models/db/workflow.py b/st2common/st2common/models/db/workflow.py index f841a182a9..7c0c51c1f2 100644 --- a/st2common/st2common/models/db/workflow.py +++ b/st2common/st2common/models/db/workflow.py @@ -38,12 +38,12 @@ class WorkflowExecutionDB(stormbase.StormFoundationDB, stormbase.ChangeRevisionF action_execution = me.StringField(required=True) spec = me.DictField() graph = me.DictField() - input = me.DictField() + input = stormbase.EscapedDictField() notify = me.DictField() context = me.DictField() - state = me.DictField() + state = stormbase.EscapedDictField() status = me.StringField(required=True) - output = me.DictField() + output = stormbase.EscapedDictField() errors = me.DynamicField() start_timestamp = db_field_types.ComplexDateTimeField(default=date_utils.get_datetime_utc_now) end_timestamp = db_field_types.ComplexDateTimeField() @@ -67,9 +67,9 @@ class TaskExecutionDB(stormbase.StormFoundationDB, stormbase.ChangeRevisionField itemized = me.BooleanField(default=False) items_count = me.IntField(min_value=0) items_concurrency = me.IntField(min_value=1) - context = me.DictField() + context = stormbase.EscapedDictField() status = me.StringField(required=True) - result = me.DictField() + result = stormbase.EscapedDictField() start_timestamp = db_field_types.ComplexDateTimeField(default=date_utils.get_datetime_utc_now) end_timestamp = db_field_types.ComplexDateTimeField() diff --git a/st2tests/integration/orquesta/test_performance.py b/st2tests/integration/orquesta/test_performance.py index 8ba8733b02..0105f99c7e 100644 --- a/st2tests/integration/orquesta/test_performance.py +++ b/st2tests/integration/orquesta/test_performance.py @@ -31,7 +31,7 @@ def test_concurrent_load(self): delay_poll = load_count * 5 wf_name = 'examples.orquesta-mock-create-vm' - wf_input = {'vm_name': 'demo1'} + wf_input = {'vm_name': 'demo1', 'meta': {'demo1.itests.org': '10.3.41.99'}} exs = [self._execute_workflow(wf_name, wf_input) for i in range(load_count)] eventlet.sleep(delay_poll) From 63b7b466f09a8512884dc18853bc9a10716725a1 Mon Sep 17 00:00:00 2001 From: W Chan Date: Tue, 25 Feb 2020 00:26:17 +0000 Subject: [PATCH 2/3] Return an error message to troubleshoot the concurrent load test --- st2tests/integration/orquesta/test_performance.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/st2tests/integration/orquesta/test_performance.py b/st2tests/integration/orquesta/test_performance.py index 0105f99c7e..cce804d5c9 100644 --- a/st2tests/integration/orquesta/test_performance.py +++ b/st2tests/integration/orquesta/test_performance.py @@ -17,6 +17,7 @@ from __future__ import absolute_import import eventlet +import json from integration.orquesta import base from six.moves import range @@ -38,7 +39,7 @@ def test_concurrent_load(self): for ex in exs: e = self._wait_for_completion(ex) - self.assertEqual(e.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) + self.assertEqual(e.status, ac_const.LIVEACTION_STATUS_SUCCEEDED, json.dumps(e.result)) self.assertIn('output', e.result) self.assertIn('vm_id', e.result['output']) From 30e42ba06bb56c0f341ba0664e750dd9790b9adf Mon Sep 17 00:00:00 2001 From: W Chan Date: Tue, 25 Feb 2020 00:48:16 +0000 Subject: [PATCH 3/3] Add step to create the virtualenv for the examples pack A test orquesta workflow is refactored to use a python action to return specific result. The use of the python action from the examples pack requires the virtualenv for the examples pack to be setup. --- scripts/travis/prepare-integration.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index 4b6f078481..4266ef7068 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -28,6 +28,9 @@ echo " === START: Catting screen process log files. ===" cat logs/screen-*.log echo " === END: Catting screen process log files. ===" +# Setup the virtualenv for the examples pack which is required for orquesta integration tests. +st2 run packs.setup_virtualenv packs=examples + # This script runs as root on Travis which means other processes which don't run # as root can't write to logs/ directory and tests fail chmod 777 logs/