From e2cca8179b0da78a766cd0dab66044128379b9c0 Mon Sep 17 00:00:00 2001 From: Jinping Han Date: Thu, 28 Feb 2019 10:43:37 -0800 Subject: [PATCH 01/13] Fixes https://github.com/StackStorm/st2/issues/4567: config_context renders against incorrect pack Root cause: When a action calls other action which belongs to the different pack, during `render_live_params`, only parent pack configuration is retrieved. Fixes: Passes action reference to `render_live_params`, if action parent pack is different with child action pack, the configuration for child pack is retired and rendered. --- st2common/st2common/services/workflows.py | 6 +++++- st2common/st2common/util/param.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/st2common/st2common/services/workflows.py b/st2common/st2common/services/workflows.py index 65a2a21040..ba237cdfce 100644 --- a/st2common/st2common/services/workflows.py +++ b/st2common/st2common/services/workflows.py @@ -535,6 +535,9 @@ def request_action_execution(wf_ex_db, task_ex_db, st2_ctx, ac_ex_req, delay=Non # Identify the runner for the action. runner_type_db = action_utils.get_runnertype_by_name(action_db.runner_type['name']) + # Identify action reference + action_ref = task_ex_db.task_spec.get('spec').get('action') + # Set context for the action execution. ac_ex_ctx = { 'pack': st2_ctx.get('pack'), @@ -545,7 +548,8 @@ def request_action_execution(wf_ex_db, task_ex_db, st2_ctx, ac_ex_req, delay=Non 'task_execution_id': str(task_ex_db.id), 'task_name': task_ex_db.task_name, 'task_id': task_ex_db.task_id - } + }, + 'ref': action_ref } if st2_ctx.get('api_user'): diff --git a/st2common/st2common/util/param.py b/st2common/st2common/util/param.py index 0dfe724123..1e738b3184 100644 --- a/st2common/st2common/util/param.py +++ b/st2common/st2common/util/param.py @@ -290,6 +290,7 @@ def render_live_params(runner_parameters, action_parameters, params, action_cont pack = action_context.get('pack') user = action_context.get('user') + action_ref = action_context.get('ref') try: config = get_config(pack, user) @@ -299,6 +300,17 @@ def render_live_params(runner_parameters, action_parameters, params, action_cont G = _create_graph(action_context, config) + # Retrieve config for action pack if it is different with parent + if action_ref: + action_pack = action_ref.split('.')[0] + if pack != action_pack: + try: + action_config = get_config(action_pack, user) + G = _create_graph(action_context, action_config) + except Exception as e: + LOG.info('Failed to retrieve action config for pack %s and user %s: %s' % ( + action_pack, user, str(e))) + # Additional contexts are applied after all other contexts (see _create_graph), but before any # of the dependencies have been resolved. for name, value in six.iteritems(additional_contexts): From a97fd8ee8737ab1c9b5acb76ca0e227c4b350baf Mon Sep 17 00:00:00 2001 From: Jinping Han Date: Thu, 28 Feb 2019 13:31:02 -0800 Subject: [PATCH 02/13] Remove render for parent pack configuration. Googed Stackstorm documentation and did not see anywhere mentioned that this is supported. Fix typo for `cancellation` --- st2common/st2common/services/workflows.py | 15 +++++++++------ st2common/st2common/util/param.py | 12 ------------ 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/st2common/st2common/services/workflows.py b/st2common/st2common/services/workflows.py index ba237cdfce..08267422e1 100644 --- a/st2common/st2common/services/workflows.py +++ b/st2common/st2common/services/workflows.py @@ -357,7 +357,7 @@ def request_resume(ac_ex_db): @retrying.retry(retry_on_exception=wf_exc.retry_on_exceptions) def request_cancellation(ac_ex_db): wf_ac_ex_id = str(ac_ex_db.id) - LOG.info('[%s] Processing cancelation request for workflow.', wf_ac_ex_id) + LOG.info('[%s] Processing cancellation request for workflow.', wf_ac_ex_id) wf_ex_dbs = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id)) @@ -388,12 +388,12 @@ def request_cancellation(ac_ex_db): root_ac_ex_db = ac_svc.get_root_execution(ac_ex_db) if root_ac_ex_db != ac_ex_db and root_ac_ex_db.status not in ac_const.LIVEACTION_CANCEL_STATES: - LOG.info('[%s] Cascading cancelation request to parent workflow.', wf_ac_ex_id) + LOG.info('[%s] Cascading cancellation request to parent workflow.', wf_ac_ex_id) root_lv_ac_db = lv_db_access.LiveAction.get(id=root_ac_ex_db.liveaction['id']) ac_svc.request_cancellation(root_lv_ac_db, None) LOG.debug('[%s] %s', wf_ac_ex_id, conductor.serialize()) - LOG.info('[%s] Completed processing cancelation request for workflow.', wf_ac_ex_id) + LOG.info('[%s] Completed processing cancellation request for workflow.', wf_ac_ex_id) return wf_ex_db @@ -535,12 +535,16 @@ def request_action_execution(wf_ex_db, task_ex_db, st2_ctx, ac_ex_req, delay=Non # Identify the runner for the action. runner_type_db = action_utils.get_runnertype_by_name(action_db.runner_type['name']) - # Identify action reference + # Identify action pack name action_ref = task_ex_db.task_spec.get('spec').get('action') + if action_ref: + pack = action_ref.split('.')[0] + else: + pack = st2_ctx.get('pack') # Set context for the action execution. ac_ex_ctx = { - 'pack': st2_ctx.get('pack'), + 'pack': pack, 'user': st2_ctx.get('user'), 'parent': st2_ctx, 'orquesta': { @@ -549,7 +553,6 @@ def request_action_execution(wf_ex_db, task_ex_db, st2_ctx, ac_ex_req, delay=Non 'task_name': task_ex_db.task_name, 'task_id': task_ex_db.task_id }, - 'ref': action_ref } if st2_ctx.get('api_user'): diff --git a/st2common/st2common/util/param.py b/st2common/st2common/util/param.py index 1e738b3184..0dfe724123 100644 --- a/st2common/st2common/util/param.py +++ b/st2common/st2common/util/param.py @@ -290,7 +290,6 @@ def render_live_params(runner_parameters, action_parameters, params, action_cont pack = action_context.get('pack') user = action_context.get('user') - action_ref = action_context.get('ref') try: config = get_config(pack, user) @@ -300,17 +299,6 @@ def render_live_params(runner_parameters, action_parameters, params, action_cont G = _create_graph(action_context, config) - # Retrieve config for action pack if it is different with parent - if action_ref: - action_pack = action_ref.split('.')[0] - if pack != action_pack: - try: - action_config = get_config(action_pack, user) - G = _create_graph(action_context, action_config) - except Exception as e: - LOG.info('Failed to retrieve action config for pack %s and user %s: %s' % ( - action_pack, user, str(e))) - # Additional contexts are applied after all other contexts (see _create_graph), but before any # of the dependencies have been resolved. for name, value in six.iteritems(additional_contexts): From 62a51fbc3d40b5e91849be82b1e2a10383e03e49 Mon Sep 17 00:00:00 2001 From: Jinping Han Date: Thu, 7 Mar 2019 11:30:18 -0800 Subject: [PATCH 03/13] New unit test for config_context renders against incorrect pack Added new unit test case for the fix. Modified two dummy packs that unit test can run. Fix code review comments. Updated CHANGELOG.txt --- CHANGELOG.rst | 2 + st2common/st2common/services/workflows.py | 13 ++-- .../tests/unit/services/test_workflow.py | 74 +++++++++++++++++++ st2tests/st2tests/base.py | 3 +- .../fixtures/packs/configs/dummy_pack_7.yaml | 2 + .../actions/render_config_context.yaml | 8 ++ .../workflows/render_config_context.yaml | 7 ++ .../packs/dummy_pack_7/actions/render.py | 7 ++ .../packs/dummy_pack_7/actions/render.yaml | 12 +++ .../packs/dummy_pack_7/config.schema.yaml | 5 ++ 10 files changed, 124 insertions(+), 9 deletions(-) create mode 100644 st2tests/st2tests/fixtures/packs/configs/dummy_pack_7.yaml create mode 100644 st2tests/st2tests/fixtures/packs/dummy_pack_20/actions/render_config_context.yaml create mode 100644 st2tests/st2tests/fixtures/packs/dummy_pack_20/actions/workflows/render_config_context.yaml create mode 100644 st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render.py create mode 100644 st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render.yaml create mode 100644 st2tests/st2tests/fixtures/packs/dummy_pack_7/config.schema.yaml diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f465bf4b52..f3ca99267b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -33,6 +33,8 @@ Fixed with items task. (bug fix) #4523 * Fix orquesta workflow bug where context variables are being overwritten on task join. (bug fix) StackStorm/orquesta#112 +* Fix `config_context` renders against incorrect pack + (bug fix) StackStorm/st2#4570 2.10.3 - March 06, 2019 ----------------------- diff --git a/st2common/st2common/services/workflows.py b/st2common/st2common/services/workflows.py index 9da0677bbc..11f6e1f3a4 100644 --- a/st2common/st2common/services/workflows.py +++ b/st2common/st2common/services/workflows.py @@ -357,7 +357,7 @@ def request_resume(ac_ex_db): @retrying.retry(retry_on_exception=wf_exc.retry_on_exceptions) def request_cancellation(ac_ex_db): wf_ac_ex_id = str(ac_ex_db.id) - LOG.info('[%s] Processing cancellation request for workflow.', wf_ac_ex_id) + LOG.info('[%s] Processing cancelation request for workflow.', wf_ac_ex_id) wf_ex_dbs = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id)) @@ -388,12 +388,12 @@ def request_cancellation(ac_ex_db): root_ac_ex_db = ac_svc.get_root_execution(ac_ex_db) if root_ac_ex_db != ac_ex_db and root_ac_ex_db.status not in ac_const.LIVEACTION_CANCEL_STATES: - LOG.info('[%s] Cascading cancellation request to parent workflow.', wf_ac_ex_id) + LOG.info('[%s] Cascading cancelation request to parent workflow.', wf_ac_ex_id) root_lv_ac_db = lv_db_access.LiveAction.get(id=root_ac_ex_db.liveaction['id']) ac_svc.request_cancellation(root_lv_ac_db, None) LOG.debug('[%s] %s', wf_ac_ex_id, conductor.serialize()) - LOG.info('[%s] Completed processing cancellation request for workflow.', wf_ac_ex_id) + LOG.info('[%s] Completed processing cancelation request for workflow.', wf_ac_ex_id) return wf_ex_db @@ -543,10 +543,7 @@ def request_action_execution(wf_ex_db, task_ex_db, st2_ctx, ac_ex_req, delay=Non # Identify action pack name action_ref = task_ex_db.task_spec.get('spec').get('action') - if action_ref: - pack = action_ref.split('.')[0] - else: - pack = st2_ctx.get('pack') + pack = action_ref.split('.')[0] if action_ref else st2_ctx.get('pack') # Set context for the action execution. ac_ex_ctx = { @@ -559,7 +556,7 @@ def request_action_execution(wf_ex_db, task_ex_db, st2_ctx, ac_ex_req, delay=Non 'task_name': task_ex_db.task_name, 'task_id': task_ex_db.task_id, 'task_route': task_ex_db.task_route - }, + } } if st2_ctx.get('api_user'): diff --git a/st2common/tests/unit/services/test_workflow.py b/st2common/tests/unit/services/test_workflow.py index d4557dc462..a32efdb4ca 100644 --- a/st2common/tests/unit/services/test_workflow.py +++ b/st2common/tests/unit/services/test_workflow.py @@ -32,8 +32,10 @@ from st2common.exceptions import action as action_exc from st2common.models.db import liveaction as lv_db_models from st2common.models.db import execution as ex_db_models +from st2common.models.db.pack import ConfigDB from st2common.persistence import execution as ex_db_access from st2common.persistence import workflow as wf_db_access +from st2common.persistence.pack import Config from st2common.services import action as action_service from st2common.services import workflows as workflow_service from st2common.transport import liveaction as lv_ac_xport @@ -44,8 +46,16 @@ TEST_PACK = 'orquesta_tests' TEST_PACK_PATH = st2tests.fixturesloader.get_fixtures_packs_base_path() + '/' + TEST_PACK +PACK_7 = 'dummy_pack_7' +PACK_7_PATH = st2tests.fixturesloader.get_fixtures_packs_base_path() + '/' + PACK_7 + +PACK_20 = 'dummy_pack_20' +PACK_20_PATH = st2tests.fixturesloader.get_fixtures_packs_base_path() + '/' + PACK_20 + PACKS = [ TEST_PACK_PATH, + PACK_7_PATH, + PACK_20_PATH, st2tests.fixturesloader.get_fixtures_packs_base_path() + '/core' ] @@ -363,3 +373,67 @@ def test_evaluate_action_execution_delay(self): ac_ex_req = {'action': 'core.noop', 'input': None, 'item_id': 1} actual_delay = workflow_service.eval_action_execution_delay(task_ex_req, ac_ex_req, True) self.assertIsNone(actual_delay) + + def test_request_action_execution_render(self): + # Manually create ConfigDB + output = 'Testing' + value = { + "config_item_one": output + } + config_db = ConfigDB(pack=PACK_7, values=value) + config = Config.add_or_update(config_db) + self.assertEqual(len(config), 3) + + wf_meta = self.get_wf_fixture_meta_data(PACK_20_PATH, 'render_config_context.yaml') + + # Manually create the liveaction and action execution objects without publishing. + lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name']) + lv_ac_db, ac_ex_db = action_service.create_request(lv_ac_db) + + # Request the workflow execution. + wf_def = self.get_wf_def(PACK_20_PATH, wf_meta) + st2_ctx = self.mock_st2_context(ac_ex_db) + wf_ex_db = workflow_service.request(wf_def, ac_ex_db, st2_ctx) + spec_module = specs_loader.get_spec_module(wf_ex_db.spec['catalog']) + wf_spec = spec_module.WorkflowSpec.deserialize(wf_ex_db.spec) + + # Pass down appropriate st2 context to the task and action execution(s). + root_st2_ctx = wf_ex_db.context.get('st2', {}) + st2_ctx = { + 'execution_id': wf_ex_db.action_execution, + 'user': root_st2_ctx.get('user'), + 'pack': root_st2_ctx.get('pack') + } + + # Manually request task execution. + task_route = 0 + task_id = 'task1' + task_spec = wf_spec.tasks.get_task(task_id) + task_ctx = {'foo': 'bar'} + + task_ex_req = { + 'id': task_id, + 'route': task_route, + 'spec': task_spec, + 'ctx': task_ctx, + 'actions': [ + {'action': 'dummy_pack_7.render', 'input': None} + ] + } + workflow_service.request_task_execution(wf_ex_db, st2_ctx, task_ex_req) + + # Check task execution is saved to the database. + task_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id)) + self.assertEqual(len(task_ex_dbs), 1) + workflow_service.request_task_execution(wf_ex_db, st2_ctx, task_ex_req) + + # Manually request action execution + task_ex_db = task_ex_dbs[0] + action_ex_db = workflow_service.request_action_execution(wf_ex_db, task_ex_db, st2_ctx, + task_ex_req['actions'][0]) + + # Check required attributes. + self.assertIsNotNone(str(action_ex_db.id)) + self.assertEqual(task_ex_db.workflow_execution, str(wf_ex_db.id)) + expected_parameters = {'value1': output} + self.assertEqual(expected_parameters, action_ex_db.parameters) diff --git a/st2tests/st2tests/base.py b/st2tests/st2tests/base.py index c53e03656a..c833c2a518 100644 --- a/st2tests/st2tests/base.py +++ b/st2tests/st2tests/base.py @@ -618,7 +618,8 @@ def mock_st2_context(self, ac_ex_db, context=None): st2_ctx = { 'st2': { 'api_url': api_util.get_full_public_api_url(), - 'action_execution_id': str(ac_ex_db.id) + 'action_execution_id': str(ac_ex_db.id), + 'user': 'stanley' } } diff --git a/st2tests/st2tests/fixtures/packs/configs/dummy_pack_7.yaml b/st2tests/st2tests/fixtures/packs/configs/dummy_pack_7.yaml new file mode 100644 index 0000000000..6b01d06715 --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/configs/dummy_pack_7.yaml @@ -0,0 +1,2 @@ +--- +config_item_one: "testing" diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_20/actions/render_config_context.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_20/actions/render_config_context.yaml new file mode 100644 index 0000000000..0f1c6582d1 --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_20/actions/render_config_context.yaml @@ -0,0 +1,8 @@ +--- +name: render_config_context +pack: dummy_pack_20 +description: Run render config context workflow +runner_type: orquesta +entry_point: workflows/render_config_context.yaml +enabled: true +parameters: {} diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_20/actions/workflows/render_config_context.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_20/actions/workflows/render_config_context.yaml new file mode 100644 index 0000000000..2d05f1dc51 --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_20/actions/workflows/render_config_context.yaml @@ -0,0 +1,7 @@ +version: 1.0 +description: Testing config context render". +tasks: + task1: + action: dummy_pack_7.render +output: + - context_value: <% task(task1).result.result.context_value %> diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render.py b/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render.py new file mode 100644 index 0000000000..97ab48fea0 --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render.py @@ -0,0 +1,7 @@ +from st2common.runners.base_action import Action + + +class PrintPythonVersionAction(Action): + + def run(self, value1): + return {"context_value": value1} diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render.yaml new file mode 100644 index 0000000000..3f11fc264b --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render.yaml @@ -0,0 +1,12 @@ +--- +name: render +runner_type: python-script +description: Action that uses config context +enabled: true +entry_point: render.py +parameters: + value1: + description: Input for action1. Defaults to config_context value. + required: false + type: "string" + default: "{{ config_context.config_item_one }}" diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_7/config.schema.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_7/config.schema.yaml new file mode 100644 index 0000000000..731f42d98c --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_7/config.schema.yaml @@ -0,0 +1,5 @@ +--- +config_item_one: + description: "Item use to test config context." + type: "string" + required: true From 0e780ad6315615d446beddedf97a55e963abf52b Mon Sep 17 00:00:00 2001 From: Jinping Han Date: Thu, 7 Mar 2019 14:59:05 -0800 Subject: [PATCH 04/13] Fix review comments for unit test. Remove action and workflow from dummy_pack_20 to orquesta_tests Rename action_ref to spec_action Rename import names for test_workflow.py and reflect changes for dummy_pack_20 --- st2common/st2common/services/workflows.py | 8 ++++---- st2common/tests/unit/services/test_workflow.py | 16 ++++++---------- .../actions/render_config_context.yaml | 2 +- .../actions/workflows/render_config_context.yaml | 0 4 files changed, 11 insertions(+), 15 deletions(-) rename st2tests/st2tests/fixtures/packs/{dummy_pack_20 => orquesta_tests}/actions/render_config_context.yaml (89%) rename st2tests/st2tests/fixtures/packs/{dummy_pack_20 => orquesta_tests}/actions/workflows/render_config_context.yaml (100%) diff --git a/st2common/st2common/services/workflows.py b/st2common/st2common/services/workflows.py index 11f6e1f3a4..7fa406345d 100644 --- a/st2common/st2common/services/workflows.py +++ b/st2common/st2common/services/workflows.py @@ -542,12 +542,12 @@ def request_action_execution(wf_ex_db, task_ex_db, st2_ctx, ac_ex_req, delay=Non runner_type_db = action_utils.get_runnertype_by_name(action_db.runner_type['name']) # Identify action pack name - action_ref = task_ex_db.task_spec.get('spec').get('action') - pack = action_ref.split('.')[0] if action_ref else st2_ctx.get('pack') + spec_action = task_ex_db.task_spec.get('spec').get('action') + pack_name = spec_action.split('.')[0] if spec_action else st2_ctx.get('pack') # Set context for the action execution. ac_ex_ctx = { - 'pack': pack, + 'pack': pack_name, 'user': st2_ctx.get('user'), 'parent': st2_ctx, 'orquesta': { @@ -1114,4 +1114,4 @@ def update_execution_records(wf_ex_db, conductor, update_lv_ac_on_states=None, # Invoke post run on the liveaction for the workflow execution. if status_changed and wf_lv_ac_db.status in ac_const.LIVEACTION_COMPLETED_STATES: LOG.info('[%s] Workflow action execution is completed and invoking post run.', wf_ac_ex_id) - runners_utils.invoke_post_run(wf_lv_ac_db) \ No newline at end of file + runners_utils.invoke_post_run(wf_lv_ac_db) diff --git a/st2common/tests/unit/services/test_workflow.py b/st2common/tests/unit/services/test_workflow.py index a32efdb4ca..dbe6e6dda7 100644 --- a/st2common/tests/unit/services/test_workflow.py +++ b/st2common/tests/unit/services/test_workflow.py @@ -32,10 +32,10 @@ from st2common.exceptions import action as action_exc from st2common.models.db import liveaction as lv_db_models from st2common.models.db import execution as ex_db_models -from st2common.models.db.pack import ConfigDB +from st2common.models.db import pack as pk_db_models from st2common.persistence import execution as ex_db_access +from st2common.persistence import pack as pk_db_access from st2common.persistence import workflow as wf_db_access -from st2common.persistence.pack import Config from st2common.services import action as action_service from st2common.services import workflows as workflow_service from st2common.transport import liveaction as lv_ac_xport @@ -49,13 +49,9 @@ PACK_7 = 'dummy_pack_7' PACK_7_PATH = st2tests.fixturesloader.get_fixtures_packs_base_path() + '/' + PACK_7 -PACK_20 = 'dummy_pack_20' -PACK_20_PATH = st2tests.fixturesloader.get_fixtures_packs_base_path() + '/' + PACK_20 - PACKS = [ TEST_PACK_PATH, PACK_7_PATH, - PACK_20_PATH, st2tests.fixturesloader.get_fixtures_packs_base_path() + '/core' ] @@ -380,18 +376,18 @@ def test_request_action_execution_render(self): value = { "config_item_one": output } - config_db = ConfigDB(pack=PACK_7, values=value) - config = Config.add_or_update(config_db) + config_db = pk_db_models.ConfigDB(pack=PACK_7, values=value) + config = pk_db_access.Config.add_or_update(config_db) self.assertEqual(len(config), 3) - wf_meta = self.get_wf_fixture_meta_data(PACK_20_PATH, 'render_config_context.yaml') + wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, 'render_config_context.yaml') # Manually create the liveaction and action execution objects without publishing. lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name']) lv_ac_db, ac_ex_db = action_service.create_request(lv_ac_db) # Request the workflow execution. - wf_def = self.get_wf_def(PACK_20_PATH, wf_meta) + wf_def = self.get_wf_def(TEST_PACK_PATH, wf_meta) st2_ctx = self.mock_st2_context(ac_ex_db) wf_ex_db = workflow_service.request(wf_def, ac_ex_db, st2_ctx) spec_module = specs_loader.get_spec_module(wf_ex_db.spec['catalog']) diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_20/actions/render_config_context.yaml b/st2tests/st2tests/fixtures/packs/orquesta_tests/actions/render_config_context.yaml similarity index 89% rename from st2tests/st2tests/fixtures/packs/dummy_pack_20/actions/render_config_context.yaml rename to st2tests/st2tests/fixtures/packs/orquesta_tests/actions/render_config_context.yaml index 0f1c6582d1..2f720cfd25 100644 --- a/st2tests/st2tests/fixtures/packs/dummy_pack_20/actions/render_config_context.yaml +++ b/st2tests/st2tests/fixtures/packs/orquesta_tests/actions/render_config_context.yaml @@ -1,6 +1,6 @@ --- name: render_config_context -pack: dummy_pack_20 +pack: orquesta_tests description: Run render config context workflow runner_type: orquesta entry_point: workflows/render_config_context.yaml diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_20/actions/workflows/render_config_context.yaml b/st2tests/st2tests/fixtures/packs/orquesta_tests/actions/workflows/render_config_context.yaml similarity index 100% rename from st2tests/st2tests/fixtures/packs/dummy_pack_20/actions/workflows/render_config_context.yaml rename to st2tests/st2tests/fixtures/packs/orquesta_tests/actions/workflows/render_config_context.yaml From 0c4115522513fb647c999000852bd64f8b424ffb Mon Sep 17 00:00:00 2001 From: Jinping Han Date: Thu, 7 Mar 2019 15:14:58 -0800 Subject: [PATCH 05/13] Remove code for getting action reference from spc.action --- st2common/st2common/services/workflows.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/st2common/st2common/services/workflows.py b/st2common/st2common/services/workflows.py index 7fa406345d..c1f7b1c675 100644 --- a/st2common/st2common/services/workflows.py +++ b/st2common/st2common/services/workflows.py @@ -542,8 +542,7 @@ def request_action_execution(wf_ex_db, task_ex_db, st2_ctx, ac_ex_req, delay=Non runner_type_db = action_utils.get_runnertype_by_name(action_db.runner_type['name']) # Identify action pack name - spec_action = task_ex_db.task_spec.get('spec').get('action') - pack_name = spec_action.split('.')[0] if spec_action else st2_ctx.get('pack') + pack_name = action_ref.split('.')[0] if action_ref else st2_ctx.get('pack') # Set context for the action execution. ac_ex_ctx = { From f76e6dd0aa7849105f286a19a73c24854081bfba Mon Sep 17 00:00:00 2001 From: Jinping Han Date: Thu, 7 Mar 2019 18:02:41 -0800 Subject: [PATCH 06/13] Integration test for config_context renders against incorrect pack In order to run integration test, test used packs has to be installed first. Install `orquesta_tests` and `dummy_pack_7` packs in `` DB Create `virtualenv` for packs Add new testcase in test_wiring.py --- st2tests/integration/orquesta/base.py | 11 +++- st2tests/integration/orquesta/test_wiring.py | 19 ++++++ st2tests/st2tests/base.py | 67 ++++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) diff --git a/st2tests/integration/orquesta/base.py b/st2tests/integration/orquesta/base.py index 4d06a9eb4e..6e5a3cc38b 100644 --- a/st2tests/integration/orquesta/base.py +++ b/st2tests/integration/orquesta/base.py @@ -25,6 +25,7 @@ from st2client import client as st2 from st2client import models from st2common.constants import action as action_constants +from st2tests.base import InstallFixturesPacks LIVEACTION_LAUNCHED_STATUSES = [ @@ -53,12 +54,20 @@ def _delete_temp_file(self, temp_file_path): os.remove(temp_file_path) -class TestWorkflowExecution(unittest2.TestCase): +class TestWorkflowExecution(InstallFixturesPacks): @classmethod def setUpClass(cls): + super(TestWorkflowExecution, cls).setUpClass() cls.st2client = st2.Client(base_url='http://127.0.0.1') + def tearDown(self): + super(TestWorkflowExecution, self).tearDown() + super(TestWorkflowExecution, self).delete_files() + + def install_packs(self, packs): + super(TestWorkflowExecution, self).install_packs(packs) + def _execute_workflow(self, action, parameters=None, execute_async=True, expected_status=None, expected_result=None): diff --git a/st2tests/integration/orquesta/test_wiring.py b/st2tests/integration/orquesta/test_wiring.py index 0306897810..70dff54b02 100644 --- a/st2tests/integration/orquesta/test_wiring.py +++ b/st2tests/integration/orquesta/test_wiring.py @@ -17,11 +17,14 @@ from __future__ import absolute_import +import mock + from integration.orquesta import base from st2common.constants import action as ac_const +@mock.patch('st2common.util.virtualenvs.BASE_PACK_REQUIREMENTS', []) class WiringTest(base.TestWorkflowExecution): def test_sequential(self): @@ -144,3 +147,19 @@ def test_output_on_error(self): self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) self.assertDictEqual(ex.result, expected_result) + + def test_config_context_renders(self): + packs = ["orquesta_tests", "dummy_pack_7"] + + self.install_packs(packs) + config_value = "testing" + wf_name = 'orquesta_tests.render_config_context' + + expected_output = {'context_value': config_value} + expected_result = {'output': expected_output} + + ex = self._execute_workflow(wf_name) + ex = self._wait_for_completion(ex) + + self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) + self.assertDictEqual(ex.result, expected_result) diff --git a/st2tests/st2tests/base.py b/st2tests/st2tests/base.py index c833c2a518..8bcd6cc865 100644 --- a/st2tests/st2tests/base.py +++ b/st2tests/st2tests/base.py @@ -47,8 +47,10 @@ from st2common.exceptions.db import StackStormDBObjectConflictError from st2common.models.db import db_setup, db_teardown, db_ensure_indexes from st2common.models.db.execution_queue import ActionExecutionSchedulingQueueItemDB +from st2common.bootstrap.actionsregistrar import ActionsRegistrar from st2common.bootstrap.base import ResourceRegistrar from st2common.bootstrap.configsregistrar import ConfigsRegistrar +from st2common.bootstrap import runnersregistrar from st2common.content.utils import get_packs_base_paths from st2common.content.loader import MetaLoader from st2common.exceptions.db import StackStormDBObjectNotFoundError @@ -58,6 +60,7 @@ from st2common.services import workflows as wf_svc from st2common.util import api as api_util from st2common.util import loader +from st2common.util.virtualenvs import setup_pack_virtualenv import st2common.models.db.rule as rule_model import st2common.models.db.rule_enforcement as rule_enforcement_model import st2common.models.db.sensor as sensor_model @@ -83,6 +86,7 @@ 'DbTestCase', 'DbModelTestCase', 'CleanDbTestCase', + 'InstallFixturesPacks', 'CleanFilesTestCase', 'IntegrationTestCase', 'RunnerTestCase', @@ -465,6 +469,69 @@ def setUp(self): self._register_pack_configs() +class InstallFixturesPacks(DbTestCase): + """ + Class installs Fixtures packs according to passed pack list + """ + def install_packs(self, packs): + self._setup_env() + self._create_pack_virtualenv(packs) + self._register_packs(packs) + self._register_pack_configs(packs) + + def delete_files(self): + try: + shutil.rmtree(self.virtualenvs_path) + except Exception: + pass + + def _setup_env(self): + self.pack_path = get_packs_base_paths()[0] + self.virtualenvs_path = os.path.join(self.pack_path, 'virtualenvs/') + + def _create_pack_virtualenv(self, packs): + self._create_virtual_path() + + for pack_name in packs: + # Create virtualenv + setup_pack_virtualenv(pack_name=pack_name, update=False, include_pip=False, + include_setuptools=False, include_wheel=False) + + def _register_packs(self, packs, validate_configs=False): + """ + Register all the packs inside the fixtures directory. + """ + # Register runners. + runnersregistrar.register_runners() + + # Register actions. + actions_registrar = ActionsRegistrar(use_pack_cache=False, fail_on_failure=True) + for pack_name in packs: + actions_registrar.register_from_pack(self.pack_path + '/' + pack_name) + + # Register packs. + registrar = ResourceRegistrar(use_pack_cache=False, use_runners_cache=True) + for pack_name in packs: + registrar._register_pack_db(pack_name=pack_name, pack_dir=self.pack_path + '/' + + pack_name) + + def _register_pack_configs(self, packs, validate_configs=False): + """ + Register all the packs inside the fixtures directory. + """ + registrar = ConfigsRegistrar(use_pack_cache=False, validate_configs=validate_configs) + for pack_name in packs: + config_path = registrar._get_config_path_for_pack(pack_name) + if os.path.exists(config_path): + registrar._register_config_for_pack(pack=pack_name, config_path=config_path) + + def _create_virtual_path(self): + if os.path.isdir(self.virtualenvs_path): + self.delete_files() + + os.mkdir(self.virtualenvs_path) + + class CleanFilesTestCase(TestCase): """ Base test class which deletes specified files and directories on setUp and `tearDown. From 82ce3c5a9fbb577f8b8b67d7ba22f09939b0bae6 Mon Sep 17 00:00:00 2001 From: Jinping Han Date: Fri, 8 Mar 2019 16:17:46 -0800 Subject: [PATCH 07/13] Remove integration test code and setup test environment from `./tools/launchdev.sh Remove integration task code made yesterday. Add code to setup integration test for render: During integration setup on cicd, `launchdev.sh start -x` will be called to install `examples` pack in order to run integration test. Add other packs are installed too, includes `core`, `default` and `pack`. Add new orquesta workflow action in examples. Still don't know which packs can be used for render action other than dummy_pack_7. --- .../actions/render_config_context.yaml | 8 +++ .../workflows/render_config_context.yaml | 7 ++ st2tests/integration/orquesta/base.py | 11 +-- st2tests/integration/orquesta/test_wiring.py | 6 +- st2tests/st2tests/base.py | 67 ------------------- tools/launchdev.sh | 6 ++ 6 files changed, 23 insertions(+), 82 deletions(-) create mode 100644 contrib/examples/actions/render_config_context.yaml create mode 100644 contrib/examples/actions/workflows/render_config_context.yaml diff --git a/contrib/examples/actions/render_config_context.yaml b/contrib/examples/actions/render_config_context.yaml new file mode 100644 index 0000000000..f9c307f130 --- /dev/null +++ b/contrib/examples/actions/render_config_context.yaml @@ -0,0 +1,8 @@ +--- +name: render_config_context +pack: examples +description: Run render config context workflow +runner_type: orquesta +entry_point: workflows/render_config_context.yaml +enabled: true +parameters: {} diff --git a/contrib/examples/actions/workflows/render_config_context.yaml b/contrib/examples/actions/workflows/render_config_context.yaml new file mode 100644 index 0000000000..2d05f1dc51 --- /dev/null +++ b/contrib/examples/actions/workflows/render_config_context.yaml @@ -0,0 +1,7 @@ +version: 1.0 +description: Testing config context render". +tasks: + task1: + action: dummy_pack_7.render +output: + - context_value: <% task(task1).result.result.context_value %> diff --git a/st2tests/integration/orquesta/base.py b/st2tests/integration/orquesta/base.py index 6e5a3cc38b..4d06a9eb4e 100644 --- a/st2tests/integration/orquesta/base.py +++ b/st2tests/integration/orquesta/base.py @@ -25,7 +25,6 @@ from st2client import client as st2 from st2client import models from st2common.constants import action as action_constants -from st2tests.base import InstallFixturesPacks LIVEACTION_LAUNCHED_STATUSES = [ @@ -54,20 +53,12 @@ def _delete_temp_file(self, temp_file_path): os.remove(temp_file_path) -class TestWorkflowExecution(InstallFixturesPacks): +class TestWorkflowExecution(unittest2.TestCase): @classmethod def setUpClass(cls): - super(TestWorkflowExecution, cls).setUpClass() cls.st2client = st2.Client(base_url='http://127.0.0.1') - def tearDown(self): - super(TestWorkflowExecution, self).tearDown() - super(TestWorkflowExecution, self).delete_files() - - def install_packs(self, packs): - super(TestWorkflowExecution, self).install_packs(packs) - def _execute_workflow(self, action, parameters=None, execute_async=True, expected_status=None, expected_result=None): diff --git a/st2tests/integration/orquesta/test_wiring.py b/st2tests/integration/orquesta/test_wiring.py index 70dff54b02..04eacbef20 100644 --- a/st2tests/integration/orquesta/test_wiring.py +++ b/st2tests/integration/orquesta/test_wiring.py @@ -24,7 +24,6 @@ from st2common.constants import action as ac_const -@mock.patch('st2common.util.virtualenvs.BASE_PACK_REQUIREMENTS', []) class WiringTest(base.TestWorkflowExecution): def test_sequential(self): @@ -149,11 +148,8 @@ def test_output_on_error(self): self.assertDictEqual(ex.result, expected_result) def test_config_context_renders(self): - packs = ["orquesta_tests", "dummy_pack_7"] - - self.install_packs(packs) config_value = "testing" - wf_name = 'orquesta_tests.render_config_context' + wf_name = 'examples.render_config_context' expected_output = {'context_value': config_value} expected_result = {'output': expected_output} diff --git a/st2tests/st2tests/base.py b/st2tests/st2tests/base.py index 8bcd6cc865..c833c2a518 100644 --- a/st2tests/st2tests/base.py +++ b/st2tests/st2tests/base.py @@ -47,10 +47,8 @@ from st2common.exceptions.db import StackStormDBObjectConflictError from st2common.models.db import db_setup, db_teardown, db_ensure_indexes from st2common.models.db.execution_queue import ActionExecutionSchedulingQueueItemDB -from st2common.bootstrap.actionsregistrar import ActionsRegistrar from st2common.bootstrap.base import ResourceRegistrar from st2common.bootstrap.configsregistrar import ConfigsRegistrar -from st2common.bootstrap import runnersregistrar from st2common.content.utils import get_packs_base_paths from st2common.content.loader import MetaLoader from st2common.exceptions.db import StackStormDBObjectNotFoundError @@ -60,7 +58,6 @@ from st2common.services import workflows as wf_svc from st2common.util import api as api_util from st2common.util import loader -from st2common.util.virtualenvs import setup_pack_virtualenv import st2common.models.db.rule as rule_model import st2common.models.db.rule_enforcement as rule_enforcement_model import st2common.models.db.sensor as sensor_model @@ -86,7 +83,6 @@ 'DbTestCase', 'DbModelTestCase', 'CleanDbTestCase', - 'InstallFixturesPacks', 'CleanFilesTestCase', 'IntegrationTestCase', 'RunnerTestCase', @@ -469,69 +465,6 @@ def setUp(self): self._register_pack_configs() -class InstallFixturesPacks(DbTestCase): - """ - Class installs Fixtures packs according to passed pack list - """ - def install_packs(self, packs): - self._setup_env() - self._create_pack_virtualenv(packs) - self._register_packs(packs) - self._register_pack_configs(packs) - - def delete_files(self): - try: - shutil.rmtree(self.virtualenvs_path) - except Exception: - pass - - def _setup_env(self): - self.pack_path = get_packs_base_paths()[0] - self.virtualenvs_path = os.path.join(self.pack_path, 'virtualenvs/') - - def _create_pack_virtualenv(self, packs): - self._create_virtual_path() - - for pack_name in packs: - # Create virtualenv - setup_pack_virtualenv(pack_name=pack_name, update=False, include_pip=False, - include_setuptools=False, include_wheel=False) - - def _register_packs(self, packs, validate_configs=False): - """ - Register all the packs inside the fixtures directory. - """ - # Register runners. - runnersregistrar.register_runners() - - # Register actions. - actions_registrar = ActionsRegistrar(use_pack_cache=False, fail_on_failure=True) - for pack_name in packs: - actions_registrar.register_from_pack(self.pack_path + '/' + pack_name) - - # Register packs. - registrar = ResourceRegistrar(use_pack_cache=False, use_runners_cache=True) - for pack_name in packs: - registrar._register_pack_db(pack_name=pack_name, pack_dir=self.pack_path + '/' + - pack_name) - - def _register_pack_configs(self, packs, validate_configs=False): - """ - Register all the packs inside the fixtures directory. - """ - registrar = ConfigsRegistrar(use_pack_cache=False, validate_configs=validate_configs) - for pack_name in packs: - config_path = registrar._get_config_path_for_pack(pack_name) - if os.path.exists(config_path): - registrar._register_config_for_pack(pack=pack_name, config_path=config_path) - - def _create_virtual_path(self): - if os.path.isdir(self.virtualenvs_path): - self.delete_files() - - os.mkdir(self.virtualenvs_path) - - class CleanFilesTestCase(TestCase): """ Base test class which deletes specified files and directories on setUp and `tearDown. diff --git a/tools/launchdev.sh b/tools/launchdev.sh index 765104ae87..fd555a2f4b 100755 --- a/tools/launchdev.sh +++ b/tools/launchdev.sh @@ -212,6 +212,8 @@ function st2start(){ if [ "$copy_examples" = true ]; then echo "Copying examples from ./contrib/examples to $PACKS_BASE_DIR" cp -Rp ./contrib/examples $PACKS_BASE_DIR + cp -Rp ./st2tests/st2tests/fixtures/packs/dummy_pack_7 $PACKS_BASE_DIR + cp -p ./st2tests/st2tests/fixtures/packs/configs/dummy_pack_7.yaml $CONFIG_BASE_DIR fi # activate virtualenv to set PYTHONPATH @@ -376,6 +378,10 @@ function st2start(){ --log-file "$LOGDIR/mistral-api.log" fi + if [ "$copy_examples" = true ]; then + st2 run packs.setup_virtualenv packs=dummy_pack_7 + fi + # Check whether screen sessions are started SCREENS=( "st2-api" From c81f04270d3f7f442652c1adf84e38287968ef95 Mon Sep 17 00:00:00 2001 From: Jinping Han Date: Sat, 9 Mar 2019 17:20:45 -0800 Subject: [PATCH 08/13] Change integration test pack from `dummy_pack_7` to `tests`. Copy dummy_pack_7 to tests pack Move setting up pack `virtualenv` to the end to make sure st2client and st2api are up running --- .../actions/workflows/render_config_context.yaml | 2 +- st2tests/tests/actions/render.py | 7 +++++++ st2tests/tests/actions/render.yaml | 12 ++++++++++++ st2tests/tests/config.schema.yaml | 5 +++++ st2tests/tests/configs/tests.yaml.yaml | 2 ++ st2tests/tests/pack.yaml | 6 ++++++ tools/launchdev.sh | 12 ++++++------ 7 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 st2tests/tests/actions/render.py create mode 100644 st2tests/tests/actions/render.yaml create mode 100644 st2tests/tests/config.schema.yaml create mode 100644 st2tests/tests/configs/tests.yaml.yaml create mode 100644 st2tests/tests/pack.yaml diff --git a/contrib/examples/actions/workflows/render_config_context.yaml b/contrib/examples/actions/workflows/render_config_context.yaml index 2d05f1dc51..3ff64f6cf2 100644 --- a/contrib/examples/actions/workflows/render_config_context.yaml +++ b/contrib/examples/actions/workflows/render_config_context.yaml @@ -2,6 +2,6 @@ version: 1.0 description: Testing config context render". tasks: task1: - action: dummy_pack_7.render + action: tests.render output: - context_value: <% task(task1).result.result.context_value %> diff --git a/st2tests/tests/actions/render.py b/st2tests/tests/actions/render.py new file mode 100644 index 0000000000..97ab48fea0 --- /dev/null +++ b/st2tests/tests/actions/render.py @@ -0,0 +1,7 @@ +from st2common.runners.base_action import Action + + +class PrintPythonVersionAction(Action): + + def run(self, value1): + return {"context_value": value1} diff --git a/st2tests/tests/actions/render.yaml b/st2tests/tests/actions/render.yaml new file mode 100644 index 0000000000..3f11fc264b --- /dev/null +++ b/st2tests/tests/actions/render.yaml @@ -0,0 +1,12 @@ +--- +name: render +runner_type: python-script +description: Action that uses config context +enabled: true +entry_point: render.py +parameters: + value1: + description: Input for action1. Defaults to config_context value. + required: false + type: "string" + default: "{{ config_context.config_item_one }}" diff --git a/st2tests/tests/config.schema.yaml b/st2tests/tests/config.schema.yaml new file mode 100644 index 0000000000..731f42d98c --- /dev/null +++ b/st2tests/tests/config.schema.yaml @@ -0,0 +1,5 @@ +--- +config_item_one: + description: "Item use to test config context." + type: "string" + required: true diff --git a/st2tests/tests/configs/tests.yaml.yaml b/st2tests/tests/configs/tests.yaml.yaml new file mode 100644 index 0000000000..6b01d06715 --- /dev/null +++ b/st2tests/tests/configs/tests.yaml.yaml @@ -0,0 +1,2 @@ +--- +config_item_one: "testing" diff --git a/st2tests/tests/pack.yaml b/st2tests/tests/pack.yaml new file mode 100644 index 0000000000..9ca7158cd0 --- /dev/null +++ b/st2tests/tests/pack.yaml @@ -0,0 +1,6 @@ +--- +name : tests +description : dummy pack +version : 0.1.0 +author : st2-dev +email : info@stackstorm.com diff --git a/tools/launchdev.sh b/tools/launchdev.sh index fd555a2f4b..e864eb72fa 100755 --- a/tools/launchdev.sh +++ b/tools/launchdev.sh @@ -212,8 +212,8 @@ function st2start(){ if [ "$copy_examples" = true ]; then echo "Copying examples from ./contrib/examples to $PACKS_BASE_DIR" cp -Rp ./contrib/examples $PACKS_BASE_DIR - cp -Rp ./st2tests/st2tests/fixtures/packs/dummy_pack_7 $PACKS_BASE_DIR - cp -p ./st2tests/st2tests/fixtures/packs/configs/dummy_pack_7.yaml $CONFIG_BASE_DIR + cp -Rp ./st2tests/tests $PACKS_BASE_DIR + cp -p ./st2tests/tests/configs/tests.yaml $CONFIG_BASE_DIR fi # activate virtualenv to set PYTHONPATH @@ -378,10 +378,6 @@ function st2start(){ --log-file "$LOGDIR/mistral-api.log" fi - if [ "$copy_examples" = true ]; then - st2 run packs.setup_virtualenv packs=dummy_pack_7 - fi - # Check whether screen sessions are started SCREENS=( "st2-api" @@ -419,6 +415,10 @@ function st2start(){ --config-file $ST2_CONF --register-all fi + if [ "$copy_examples" = true ]; then + st2 run packs.setup_virtualenv packs=tests + fi + # List screen sessions screen -ls || exit 0 } From 6f62b3c9325676848ac77951fb798556b4cc892f Mon Sep 17 00:00:00 2001 From: Jinping Han Date: Tue, 12 Mar 2019 14:43:52 -0700 Subject: [PATCH 09/13] Move tests pack from st2 repo to st2test repo. 1. Move tests pack action to https://github.com/StackStorm/st2tests/tree/master/packs/tests Add a default value to config.schema.yaml and renamed action `render` to `render_config_context` Please see https://github.com/StackStorm/st2tests/pull/150 2. Made change in CHANGELOG.rst 3. Rename copy_examples to copy_test_packs in `tools/launchdev.sh` 4. Handle gracefully when authentication is on for setting up virtualenv. 5. Renamed action `render` to `render_config_context` for unit test dummy_pack_7 --- CHANGELOG.rst | 4 +-- .../workflows/render_config_context.yaml | 2 +- .../tests/unit/services/test_workflow.py | 2 +- st2tests/integration/orquesta/test_wiring.py | 2 +- .../{render.py => render_config_context.py} | 0 ...render.yaml => render_config_context.yaml} | 6 ++-- .../workflows/render_config_context.yaml | 2 +- st2tests/tests/actions/render.py | 7 ----- st2tests/tests/actions/render.yaml | 12 ------- st2tests/tests/config.schema.yaml | 5 --- st2tests/tests/configs/tests.yaml.yaml | 2 -- st2tests/tests/pack.yaml | 6 ---- tools/launchdev.sh | 31 ++++++++++++++----- 13 files changed, 32 insertions(+), 49 deletions(-) rename st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/{render.py => render_config_context.py} (100%) rename st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/{render.yaml => render_config_context.yaml} (57%) delete mode 100644 st2tests/tests/actions/render.py delete mode 100644 st2tests/tests/actions/render.yaml delete mode 100644 st2tests/tests/config.schema.yaml delete mode 100644 st2tests/tests/configs/tests.yaml.yaml delete mode 100644 st2tests/tests/pack.yaml diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f3ca99267b..5dd1570a41 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -33,8 +33,8 @@ Fixed with items task. (bug fix) #4523 * Fix orquesta workflow bug where context variables are being overwritten on task join. (bug fix) StackStorm/orquesta#112 -* Fix `config_context` renders against incorrect pack - (bug fix) StackStorm/st2#4570 +* Fix rendering of config_context in orquesta task that references action in different pack + (bug fix) #4570 2.10.3 - March 06, 2019 ----------------------- diff --git a/contrib/examples/actions/workflows/render_config_context.yaml b/contrib/examples/actions/workflows/render_config_context.yaml index 3ff64f6cf2..24f2d5394d 100644 --- a/contrib/examples/actions/workflows/render_config_context.yaml +++ b/contrib/examples/actions/workflows/render_config_context.yaml @@ -2,6 +2,6 @@ version: 1.0 description: Testing config context render". tasks: task1: - action: tests.render + action: tests.render_config_context output: - context_value: <% task(task1).result.result.context_value %> diff --git a/st2common/tests/unit/services/test_workflow.py b/st2common/tests/unit/services/test_workflow.py index dbe6e6dda7..09e7b2eda2 100644 --- a/st2common/tests/unit/services/test_workflow.py +++ b/st2common/tests/unit/services/test_workflow.py @@ -413,7 +413,7 @@ def test_request_action_execution_render(self): 'spec': task_spec, 'ctx': task_ctx, 'actions': [ - {'action': 'dummy_pack_7.render', 'input': None} + {'action': 'dummy_pack_7.render_config_context', 'input': None} ] } workflow_service.request_task_execution(wf_ex_db, st2_ctx, task_ex_req) diff --git a/st2tests/integration/orquesta/test_wiring.py b/st2tests/integration/orquesta/test_wiring.py index 04eacbef20..297a171bb0 100644 --- a/st2tests/integration/orquesta/test_wiring.py +++ b/st2tests/integration/orquesta/test_wiring.py @@ -148,7 +148,7 @@ def test_output_on_error(self): self.assertDictEqual(ex.result, expected_result) def test_config_context_renders(self): - config_value = "testing" + config_value = "Testing" wf_name = 'examples.render_config_context' expected_output = {'context_value': config_value} diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render.py b/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render_config_context.py similarity index 100% rename from st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render.py rename to st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render_config_context.py diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render_config_context.yaml similarity index 57% rename from st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render.yaml rename to st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render_config_context.yaml index 3f11fc264b..e67d323c65 100644 --- a/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render.yaml +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render_config_context.yaml @@ -1,12 +1,12 @@ --- -name: render +name: render_config_context runner_type: python-script description: Action that uses config context enabled: true -entry_point: render.py +entry_point: render_config_context.py parameters: value1: - description: Input for action1. Defaults to config_context value. + description: Input for render_config_context. Defaults to config_context value. required: false type: "string" default: "{{ config_context.config_item_one }}" diff --git a/st2tests/st2tests/fixtures/packs/orquesta_tests/actions/workflows/render_config_context.yaml b/st2tests/st2tests/fixtures/packs/orquesta_tests/actions/workflows/render_config_context.yaml index 2d05f1dc51..5683cbe98f 100644 --- a/st2tests/st2tests/fixtures/packs/orquesta_tests/actions/workflows/render_config_context.yaml +++ b/st2tests/st2tests/fixtures/packs/orquesta_tests/actions/workflows/render_config_context.yaml @@ -2,6 +2,6 @@ version: 1.0 description: Testing config context render". tasks: task1: - action: dummy_pack_7.render + action: dummy_pack_7.render_config_context output: - context_value: <% task(task1).result.result.context_value %> diff --git a/st2tests/tests/actions/render.py b/st2tests/tests/actions/render.py deleted file mode 100644 index 97ab48fea0..0000000000 --- a/st2tests/tests/actions/render.py +++ /dev/null @@ -1,7 +0,0 @@ -from st2common.runners.base_action import Action - - -class PrintPythonVersionAction(Action): - - def run(self, value1): - return {"context_value": value1} diff --git a/st2tests/tests/actions/render.yaml b/st2tests/tests/actions/render.yaml deleted file mode 100644 index 3f11fc264b..0000000000 --- a/st2tests/tests/actions/render.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -name: render -runner_type: python-script -description: Action that uses config context -enabled: true -entry_point: render.py -parameters: - value1: - description: Input for action1. Defaults to config_context value. - required: false - type: "string" - default: "{{ config_context.config_item_one }}" diff --git a/st2tests/tests/config.schema.yaml b/st2tests/tests/config.schema.yaml deleted file mode 100644 index 731f42d98c..0000000000 --- a/st2tests/tests/config.schema.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -config_item_one: - description: "Item use to test config context." - type: "string" - required: true diff --git a/st2tests/tests/configs/tests.yaml.yaml b/st2tests/tests/configs/tests.yaml.yaml deleted file mode 100644 index 6b01d06715..0000000000 --- a/st2tests/tests/configs/tests.yaml.yaml +++ /dev/null @@ -1,2 +0,0 @@ ---- -config_item_one: "testing" diff --git a/st2tests/tests/pack.yaml b/st2tests/tests/pack.yaml deleted file mode 100644 index 9ca7158cd0..0000000000 --- a/st2tests/tests/pack.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -name : tests -description : dummy pack -version : 0.1.0 -author : st2-dev -email : info@stackstorm.com diff --git a/tools/launchdev.sh b/tools/launchdev.sh index e864eb72fa..41bc413954 100755 --- a/tools/launchdev.sh +++ b/tools/launchdev.sh @@ -9,7 +9,7 @@ runner_count=1 scheduler_count=1 workflow_engine_count=1 use_gunicorn=true -copy_examples=false +copy_test_packs=false load_content=true use_ipv6=false include_mistral=false @@ -29,7 +29,7 @@ while getopts ":r:s:w:gxcu6m" o; do use_gunicorn=false ;; x) - copy_examples=true + copy_test_packs=true ;; c) load_content=false @@ -209,11 +209,10 @@ function st2start(){ cp -Rp ./contrib/core/ $PACKS_BASE_DIR cp -Rp ./contrib/packs/ $PACKS_BASE_DIR - if [ "$copy_examples" = true ]; then - echo "Copying examples from ./contrib/examples to $PACKS_BASE_DIR" + if [ "$copy_test_packs" = true ]; then + echo "Copying test packs examples and tests to $PACKS_BASE_DIR" cp -Rp ./contrib/examples $PACKS_BASE_DIR - cp -Rp ./st2tests/tests $PACKS_BASE_DIR - cp -p ./st2tests/tests/configs/tests.yaml $CONFIG_BASE_DIR + cp -Rp ./st2tests/packs/tests $PACKS_BASE_DIR fi # activate virtualenv to set PYTHONPATH @@ -415,8 +414,24 @@ function st2start(){ --config-file $ST2_CONF --register-all fi - if [ "$copy_examples" = true ]; then - st2 run packs.setup_virtualenv packs=tests + if [ "$copy_test_packs" = true ]; then + # Check if authentication is enabled + while read -r line; do + if [ "$line" == "[auth]" ]; then + found_auth=true + fi + + if [ "$found_auth" = true ]; then + typeset -l line + if [ "$line" = "enable = true" ]; then + echo "Warning: Please setup virtualenv for pack \"tests\" before run integration test" + break + elif [ "$line" = "enable = false" ]; then + st2 run packs.setup_virtualenv packs=tests + break + fi + fi + done < "$ST2_CONF" fi # List screen sessions From eaa3858f5eaacdb7ef8d74bf6c7ca927c5b1e59a Mon Sep 17 00:00:00 2001 From: Jinping Han Date: Tue, 12 Mar 2019 16:07:15 -0700 Subject: [PATCH 10/13] Add step to clone st2tests repo and copy tests pack Clone st2tests repo to /tmp directory Copy st2tests/tests pack to /opt/stackstorm/packs Remove /tmp/st2tests directory --- tools/launchdev.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/launchdev.sh b/tools/launchdev.sh index 41bc413954..d74bfbe59f 100755 --- a/tools/launchdev.sh +++ b/tools/launchdev.sh @@ -212,7 +212,17 @@ function st2start(){ if [ "$copy_test_packs" = true ]; then echo "Copying test packs examples and tests to $PACKS_BASE_DIR" cp -Rp ./contrib/examples $PACKS_BASE_DIR - cp -Rp ./st2tests/packs/tests $PACKS_BASE_DIR + # Clone st2tests in /tmp directory. + pushd /tmp + git clone https://github.com/StackStorm/st2tests.git + ret=$? + if [ ${ret} -eq 0 ]; then + cp -Rp ./st2tests/packs/tests $PACKS_BASE_DIR + rm -R st2tests/ + else + echo "Failed to clone st2tests repo" + fi + popd fi # activate virtualenv to set PYTHONPATH From c1f0724008d5eaa90b61f78e644fc2e527dcd0c3 Mon Sep 17 00:00:00 2001 From: Jinping Han Date: Tue, 12 Mar 2019 16:16:31 -0700 Subject: [PATCH 11/13] Remove unused import: `mock` --- st2tests/integration/orquesta/test_wiring.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/st2tests/integration/orquesta/test_wiring.py b/st2tests/integration/orquesta/test_wiring.py index 297a171bb0..e62e957377 100644 --- a/st2tests/integration/orquesta/test_wiring.py +++ b/st2tests/integration/orquesta/test_wiring.py @@ -17,8 +17,6 @@ from __future__ import absolute_import -import mock - from integration.orquesta import base from st2common.constants import action as ac_const From 82fcc34880a6d34fdba304e88a3d73f10db3b6ac Mon Sep 17 00:00:00 2001 From: Jinping Han Date: Wed, 13 Mar 2019 11:30:59 -0700 Subject: [PATCH 12/13] Remove authentication setup checking for simplicity If `st2 run packs.setup_virtualenv packs=tests` returns an error, then warn users --- tools/launchdev.sh | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/tools/launchdev.sh b/tools/launchdev.sh index d74bfbe59f..31824777e4 100755 --- a/tools/launchdev.sh +++ b/tools/launchdev.sh @@ -425,23 +425,10 @@ function st2start(){ fi if [ "$copy_test_packs" = true ]; then - # Check if authentication is enabled - while read -r line; do - if [ "$line" == "[auth]" ]; then - found_auth=true - fi - - if [ "$found_auth" = true ]; then - typeset -l line - if [ "$line" = "enable = true" ]; then - echo "Warning: Please setup virtualenv for pack \"tests\" before run integration test" - break - elif [ "$line" = "enable = false" ]; then - st2 run packs.setup_virtualenv packs=tests - break - fi - fi - done < "$ST2_CONF" + st2 run packs.setup_virtualenv packs=tests + if [ $? != 0 ]; then + echo "Warning: Please setup virtualenv for pack \"tests\" before run integration test" + fi fi # List screen sessions From c81e6d12e9bc3fea674813675749caa1e3272414 Mon Sep 17 00:00:00 2001 From: Jinping Han Date: Wed, 13 Mar 2019 11:43:11 -0700 Subject: [PATCH 13/13] Change warning message. Change the message to Unable to setup virtualenv for the \"tests\" pack. Please setup virtualenv for the \"tests\" pack before running integration tests. --- tools/launchdev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/launchdev.sh b/tools/launchdev.sh index 31824777e4..1f37336ea8 100755 --- a/tools/launchdev.sh +++ b/tools/launchdev.sh @@ -427,7 +427,7 @@ function st2start(){ if [ "$copy_test_packs" = true ]; then st2 run packs.setup_virtualenv packs=tests if [ $? != 0 ]; then - echo "Warning: Please setup virtualenv for pack \"tests\" before run integration test" + echo "Warning: Unable to setup virtualenv for the \"tests\" pack. Please setup virtualenv for the \"tests\" pack before running integration tests" fi fi