diff --git a/st2api/st2api/controllers/v1/actions.py b/st2api/st2api/controllers/v1/actions.py index 88e5993e49..6dddb35314 100644 --- a/st2api/st2api/controllers/v1/actions.py +++ b/st2api/st2api/controllers/v1/actions.py @@ -17,6 +17,7 @@ import os.path import stat import errno +import uuid import six from mongoengine import ValidationError @@ -31,18 +32,26 @@ from st2common.constants.triggers import ACTION_FILE_WRITTEN_TRIGGER from st2common.exceptions.action import InvalidActionParameterException from st2common.exceptions.apivalidation import ValueValidationException +from st2common.exceptions.rbac import ResourceAccessDeniedError from st2common.persistence.action import Action from st2common.models.api.action import ActionAPI from st2common.persistence.pack import Pack from st2common.rbac.types import PermissionType from st2common.rbac.backends import get_rbac_backend from st2common.router import abort +from st2common.router import GenericRequestParam from st2common.router import Response from st2common.validators.api.misc import validate_not_part_of_system_pack +from st2common.validators.api.misc import validate_not_part_of_system_pack_by_name from st2common.content.utils import get_pack_base_path from st2common.content.utils import get_pack_resource_file_abs_path from st2common.content.utils import get_relative_path_to_pack_file from st2common.services.packs import delete_action_files_from_pack +from st2common.services.packs import clone_action_files +from st2common.services.packs import clone_action_db +from st2common.services.packs import temp_backup_action_files +from st2common.services.packs import remove_temp_action_files +from st2common.services.packs import restore_temp_action_files from st2common.transport.reactor import TriggerDispatcher from st2common.util.system_info import get_host_info import st2common.validators.api.action as action_validator @@ -281,6 +290,146 @@ def delete(self, options, ref_or_id, requester_user): LOG.audit("Action deleted. Action.id=%s" % (action_db.id), extra=extra) return Response(status=http_client.NO_CONTENT) + def clone(self, dest_data, ref_or_id, requester_user): + """ + Clone an action from source pack to destination pack. + Handles requests: + POST /actions/{ref_or_id}/clone + """ + + source_action_db = self._get_by_ref_or_id(ref_or_id=ref_or_id) + if not source_action_db: + msg = "The requested source for cloning operation doesn't exists" + abort(http_client.BAD_REQUEST, six.text_type(msg)) + + extra = {"action_db": source_action_db} + LOG.audit( + "Source action found. Action.id=%s" % (source_action_db.id), extra=extra + ) + + try: + permission_type = PermissionType.ACTION_VIEW + rbac_utils = get_rbac_backend().get_utils_class() + rbac_utils.assert_user_has_resource_db_permission( + user_db=requester_user, + resource_db=source_action_db, + permission_type=permission_type, + ) + except ResourceAccessDeniedError as e: + abort(http_client.UNAUTHORIZED, six.text_type(e)) + + cloned_dest_action_db = clone_action_db( + source_action_db=source_action_db, + dest_pack=dest_data.dest_pack, + dest_action=dest_data.dest_action, + ) + + cloned_action_api = ActionAPI.from_model(cloned_dest_action_db) + + try: + permission_type = PermissionType.ACTION_CREATE + rbac_utils.assert_user_has_resource_api_permission( + user_db=requester_user, + resource_api=cloned_action_api, + permission_type=permission_type, + ) + except ResourceAccessDeniedError as e: + abort(http_client.UNAUTHORIZED, six.text_type(e)) + + dest_pack_base_path = get_pack_base_path(pack_name=dest_data.dest_pack) + + if not os.path.isdir(dest_pack_base_path): + msg = "Destination pack '%s' doesn't exist" % (dest_data.dest_pack) + abort(http_client.BAD_REQUEST, six.text_type(msg)) + + dest_pack_base_path = get_pack_base_path(pack_name=dest_data.dest_pack) + dest_ref = ".".join([dest_data.dest_pack, dest_data.dest_action]) + dest_action_db = self._get_by_ref(resource_ref=dest_ref) + + try: + validate_not_part_of_system_pack_by_name(dest_data.dest_pack) + except ValueValidationException as e: + abort(http_client.BAD_REQUEST, six.text_type(e)) + + if dest_action_db: + if not dest_data.overwrite: + msg = "The requested destination action already exists" + abort(http_client.BAD_REQUEST, six.text_type(msg)) + + try: + permission_type = PermissionType.ACTION_DELETE + rbac_utils.assert_user_has_resource_db_permission( + user_db=requester_user, + resource_db=dest_action_db, + permission_type=permission_type, + ) + options = GenericRequestParam(remove_files=True) + dest_metadata_file = dest_action_db["metadata_file"] + dest_entry_point = dest_action_db["entry_point"] + temp_sub_dir = str(uuid.uuid4()) + temp_backup_action_files( + dest_pack_base_path, + dest_metadata_file, + dest_entry_point, + temp_sub_dir, + ) + self.delete(options, dest_ref, requester_user) + except ResourceAccessDeniedError as e: + abort(http_client.UNAUTHORIZED, six.text_type(e)) + except Exception as e: + LOG.debug( + "Exception encountered during deleting existing destination action. " + "Exception was: %s", + e, + ) + abort(http_client.INTERNAL_SERVER_ERROR, six.text_type(e)) + + try: + post_response = self.post(cloned_action_api, requester_user) + if post_response.status_code != http_client.CREATED: + raise Exception("Could not add cloned action to database.") + cloned_dest_action_db["id"] = post_response.json["id"] + clone_action_files( + source_action_db=source_action_db, + dest_action_db=cloned_dest_action_db, + dest_pack_base_path=dest_pack_base_path, + ) + extra = {"cloned_acion_db": cloned_dest_action_db} + LOG.audit( + "Action cloned. Action.id=%s" % (cloned_dest_action_db.id), extra=extra + ) + if dest_action_db: + remove_temp_action_files(temp_sub_dir) + return post_response + except PermissionError as e: + LOG.error("No permission to clone the action. Exception was %s", e) + delete_action_files_from_pack( + pack_name=cloned_dest_action_db["pack"], + entry_point=cloned_dest_action_db["entry_point"], + metadata_file=cloned_dest_action_db["metadata_file"], + ) + if post_response.status_code == http_client.CREATED: + Action.delete(cloned_dest_action_db) + if dest_action_db: + self._restore_action(dest_action_db, dest_pack_base_path, temp_sub_dir) + abort(http_client.FORBIDDEN, six.text_type(e)) + except Exception as e: + LOG.error( + "Exception encountered during cloning action. Exception was %s", + e, + ) + delete_action_files_from_pack( + pack_name=cloned_dest_action_db["pack"], + entry_point=cloned_dest_action_db["entry_point"], + metadata_file=cloned_dest_action_db["metadata_file"], + ) + if post_response.status_code == http_client.CREATED: + Action.delete(cloned_dest_action_db) + if dest_action_db: + self._restore_action(dest_action_db, dest_pack_base_path, temp_sub_dir) + + abort(http_client.INTERNAL_SERVER_ERROR, six.text_type(e)) + def _handle_data_files(self, pack_ref, data_files): """ Method for handling action data files. @@ -392,5 +541,16 @@ def _dispatch_trigger_for_written_data_files(self, action_db, written_data_files } self._trigger_dispatcher.dispatch(trigger=trigger, payload=payload) + def _restore_action(self, action_db, pack_base_path, temp_sub_dir): + restore_temp_action_files( + pack_base_path, + action_db["metadata_file"], + action_db["entry_point"], + temp_sub_dir, + ) + action_db.id = None + Action.add_or_update(action_db) + remove_temp_action_files(temp_sub_dir) + actions_controller = ActionsController() diff --git a/st2api/tests/unit/controllers/v1/test_actions.py b/st2api/tests/unit/controllers/v1/test_actions.py index f28db5d52f..8225af15e2 100644 --- a/st2api/tests/unit/controllers/v1/test_actions.py +++ b/st2api/tests/unit/controllers/v1/test_actions.py @@ -262,6 +262,35 @@ }, } +ACTION_16 = { + "name": "st2.dummy.source_action", + "description": "test description", + "enabled": True, + "pack": "sourcepack", + "entry_point": "/tmp/test/source_action.py", + "runner_type": "python-script", + "parameters": { + "x": {"type": "string", "default": "X1"}, + "y": {"type": "string", "default": "Y1"}, + }, + "tags": [ + {"name": "tag1", "value": "dont-care1"}, + {"name": "tag2", "value": "dont-care2"}, + ], +} + +ACTION_17 = { + "name": "st2.dummy.clone_action", + "description": "test description", + "enabled": True, + "pack": "clonepack", + "entry_point": "/tmp/test/clone_action.sh", + "runner_type": "local-shell-script", + "parameters": { + "a": {"type": "string", "default": "A1"}, + }, +} + ACTION_WITH_NOTIFY = { "name": "st2.dummy.action_notify_test", "description": "test description", @@ -852,6 +881,229 @@ def test_delete_action_belonging_to_system_pack(self): del_resp = self.__do_delete(action_id, expect_errors=True) self.assertEqual(del_resp.status_int, 400) + @mock.patch.object(os.path, "isdir", mock.MagicMock(return_value=True)) + @mock.patch("st2api.controllers.v1.actions.clone_action_files") + @mock.patch.object( + action_validator, "validate_action", mock.MagicMock(return_value=True) + ) + def test_clone(self, mock_clone_action): + source_post_resp = self.__do_post(ACTION_16) + self.assertEqual(source_post_resp.status_int, 201) + dest_data_body = { + "dest_pack": ACTION_17["pack"], + "dest_action": ACTION_17["name"], + } + source_ref_or_id = "%s.%s" % (ACTION_16["pack"], ACTION_16["name"]) + clone_resp = self.__do_clone(dest_data_body, source_ref_or_id) + self.assertEqual(clone_resp.status_int, 201) + get_resp = self.__do_get_actions_by_url_parameter("name", ACTION_17["name"]) + self.assertEqual(get_resp.status_int, 200) + self.assertTrue(mock_clone_action.called) + self.__do_delete(self.__get_action_id(source_post_resp)) + self.__do_delete(self.__get_action_id(clone_resp)) + + @mock.patch.object(os.path, "isdir", mock.MagicMock(return_value=True)) + @mock.patch("st2api.controllers.v1.actions.remove_temp_action_files") + @mock.patch("st2api.controllers.v1.actions.temp_backup_action_files") + @mock.patch("st2api.controllers.v1.actions.clone_action_files") + @mock.patch.object( + action_validator, "validate_action", mock.MagicMock(return_value=True) + ) + def test_clone_overwrite( + self, mock_clone_action, mock_temp_backup, mock_clean_backup + ): + source_post_resp = self.__do_post(ACTION_16) + self.assertEqual(source_post_resp.status_int, 201) + dest_post_resp = self.__do_post(ACTION_17) + self.assertEqual(dest_post_resp.status_int, 201) + dest_data_body = { + "dest_pack": ACTION_17["pack"], + "dest_action": ACTION_17["name"], + "overwrite": True, + } + source_ref_or_id = "%s.%s" % (ACTION_16["pack"], ACTION_16["name"]) + clone_resp = self.__do_clone(dest_data_body, source_ref_or_id) + self.assertEqual(clone_resp.status_int, 201) + get_resp = self.__do_get_actions_by_url_parameter("name", ACTION_17["name"]) + expected_params_dict = ACTION_16["parameters"] + actual_prams_dict = get_resp.json[0]["parameters"] + self.assertDictEqual(actual_prams_dict, expected_params_dict) + actual_runner_type = get_resp.json[0]["runner_type"] + self.assertNotEqual(actual_runner_type, ACTION_17["runner_type"]) + self.assertTrue(mock_clone_action.called) + self.assertTrue(mock_temp_backup.called) + self.assertTrue(mock_clean_backup.called) + self.__do_delete(self.__get_action_id(source_post_resp)) + self.__do_delete(self.__get_action_id(clone_resp)) + + @mock.patch.object( + action_validator, "validate_action", mock.MagicMock(return_value=True) + ) + def test_clone_source_does_not_exist(self): + dest_data_body = { + "dest_pack": ACTION_17["pack"], + "dest_action": ACTION_17["name"], + } + source_ref_or_id = "%s.%s" % (ACTION_16["pack"], ACTION_16["name"]) + clone_resp = self.__do_clone( + dest_data_body, + source_ref_or_id, + expect_errors=True, + ) + # clone operation failed and asserting response status code and error msg + self.assertEqual(clone_resp.status_int, 404) + msg = 'Resource with a reference or id "%s" not found' % source_ref_or_id + self.assertEqual(clone_resp.json["faultstring"], msg) + + @mock.patch.object( + action_validator, "validate_action", mock.MagicMock(return_value=True) + ) + def test_clone_destination_pack_does_not_exist(self): + source_post_resp = self.__do_post(ACTION_16) + dest_data_body = { + "dest_pack": ACTION_17["pack"], + "dest_action": ACTION_17["name"], + } + source_ref_or_id = "%s.%s" % (ACTION_16["pack"], ACTION_16["name"]) + clone_resp = self.__do_clone( + dest_data_body, + source_ref_or_id, + expect_errors=True, + ) + self.assertEqual(clone_resp.status_int, 400) + msg = "Destination pack '%s' doesn't exist" % ACTION_17["pack"] + self.assertEqual(clone_resp.json["faultstring"], msg) + self.__do_delete(self.__get_action_id(source_post_resp)) + + @mock.patch.object(os.path, "isdir", mock.MagicMock(return_value=True)) + @mock.patch.object( + action_validator, "validate_action", mock.MagicMock(return_value=True) + ) + def test_clone_destination_action_already_exist(self): + source_post_resp = self.__do_post(ACTION_16) + dest_post_resp = self.__do_post(ACTION_17) + dest_data_body = { + "dest_pack": ACTION_17["pack"], + "dest_action": ACTION_17["name"], + } + source_ref_or_id = "%s.%s" % (ACTION_16["pack"], ACTION_16["name"]) + clone_resp = self.__do_clone( + dest_data_body, + source_ref_or_id, + expect_errors=True, + ) + self.assertEqual(clone_resp.status_int, 400) + msg = "The requested destination action already exists" + self.assertEqual(clone_resp.json["faultstring"], msg) + self.__do_delete(self.__get_action_id(source_post_resp)) + self.__do_delete(self.__get_action_id(dest_post_resp)) + + @mock.patch.object(os.path, "isdir", mock.MagicMock(return_value=True)) + @mock.patch("st2api.controllers.v1.actions.clone_action_files") + @mock.patch.object( + action_validator, "validate_action", mock.MagicMock(return_value=True) + ) + def test_clone_permission_error(self, mock_clone_action): + msg = "No permission to access the files for cloning operation" + mock_clone_action.side_effect = PermissionError(msg) + source_post_resp = self.__do_post(ACTION_16) + dest_data_body = { + "dest_pack": ACTION_17["pack"], + "dest_action": "clone_action_3", + } + source_ref_or_id = "%s.%s" % (ACTION_16["pack"], ACTION_16["name"]) + clone_resp = self.__do_clone( + dest_data_body, + source_ref_or_id, + expect_errors=True, + ) + self.assertEqual(clone_resp.status_int, 403) + self.assertEqual(clone_resp.json["faultstring"], msg) + self.__do_delete(self.__get_action_id(source_post_resp)) + + @mock.patch.object(os.path, "isdir", mock.MagicMock(return_value=True)) + @mock.patch("st2api.controllers.v1.actions.delete_action_files_from_pack") + @mock.patch("st2api.controllers.v1.actions.clone_action_files") + @mock.patch.object( + action_validator, "validate_action", mock.MagicMock(return_value=True) + ) + def test_clone_exception(self, mock_clone_action, mock_delete_files): + msg = "Exception encountered during cloning action." + mock_clone_action.side_effect = Exception(msg) + source_post_resp = self.__do_post(ACTION_16) + dest_data_body = { + "dest_pack": ACTION_17["pack"], + "dest_action": "clone_action_4", + } + source_ref_or_id = "%s.%s" % (ACTION_16["pack"], ACTION_16["name"]) + clone_resp = self.__do_clone( + dest_data_body, + source_ref_or_id, + expect_errors=True, + ) + self.assertEqual(clone_resp.status_int, 500) + self.assertEqual(clone_resp.json["faultstring"], msg) + # asserting delete_action_files_from_pack function called i.e. cloned files are cleaned up + self.assertTrue(mock_delete_files.called) + self.__do_delete(self.__get_action_id(source_post_resp)) + + @mock.patch.object(os.path, "isdir", mock.MagicMock(return_value=True)) + @mock.patch("st2api.controllers.v1.actions.delete_action_files_from_pack") + @mock.patch("st2api.controllers.v1.actions.remove_temp_action_files") + @mock.patch("st2api.controllers.v1.actions.restore_temp_action_files") + @mock.patch("st2api.controllers.v1.actions.temp_backup_action_files") + @mock.patch("st2api.controllers.v1.actions.clone_action_files") + @mock.patch.object( + action_validator, "validate_action", mock.MagicMock(return_value=True) + ) + def test_clone_overwrite_exception_destination_recovered( + self, + mock_clone_overwrite, + mock_backup_files, + mock_restore_files, + mock_remove_backup, + mock_clean_files, + ): + msg = "Exception encountered during overwriting action." + mock_clone_overwrite.side_effect = Exception(msg) + source_post_resp = self.__do_post(ACTION_16) + self.__do_post(ACTION_17) + dest_data_body = { + "dest_pack": ACTION_17["pack"], + "dest_action": ACTION_17["name"], + "overwrite": True, + } + source_ref_or_id = "%s.%s" % (ACTION_16["pack"], ACTION_16["name"]) + clone_resp = self.__do_clone( + dest_data_body, + source_ref_or_id, + expect_errors=True, + ) + self.assertEqual(clone_resp.status_int, 500) + # asserting temp_backup_action_files function called + self.assertTrue(mock_backup_files.called) + # asserting restore_temp_action_files called i.e. original ACTION_17 restored + self.assertTrue(mock_restore_files.called) + # asserting remove_temp_action_files function called + self.assertTrue(mock_remove_backup.called) + # asserting delete_action_files_from_pack called i.e. cloned files are cleaned up + self.assertTrue(mock_clean_files.called) + # retrieving oringinal ACTION_17 from db which is reregistered after exception + dest_get_resp = self.__do_get_actions_by_url_parameter( + "name", ACTION_17["name"] + ) + self.assertEqual(dest_get_resp.status_int, 200) + expected_runner_type = ACTION_17["runner_type"] + actual_runner_type = dest_get_resp.json[0]["runner_type"] + # asserting ACTION_17 has original runner type + self.assertEqual(actual_runner_type, expected_runner_type) + expeted_parameters = ACTION_17["parameters"] + actual_parameters = dest_get_resp.json[0]["parameters"] + # asserting ACTION_17 has original parameters + self.assertDictEqual(actual_parameters, expeted_parameters) + self.__do_delete(self.__get_action_id(source_post_resp)) + self.__do_delete(dest_get_resp.json[0]["id"]) + def _insert_mock_models(self): action_1_id = self.__get_action_id(self.__do_post(ACTION_1)) action_2_id = self.__get_action_id(self.__do_post(ACTION_2)) @@ -904,3 +1156,15 @@ def __do_delete_action_with_files(self, options, action_id, expect_errors=False) options, expect_errors=expect_errors, ) + + def __do_clone( + self, + dest_data, + ref_or_id, + expect_errors=False, + ): + return self.app.post_json( + "/v1/actions/%s/clone" % (ref_or_id), + dest_data, + expect_errors=expect_errors, + ) diff --git a/st2client/st2client/commands/action.py b/st2client/st2client/commands/action.py index 41fca2da88..6e2478fe13 100644 --- a/st2client/st2client/commands/action.py +++ b/st2client/st2client/commands/action.py @@ -208,6 +208,7 @@ def __init__(self, description, app, subparsers, parent_parser=None): "get": ActionGetCommand, "update": ActionUpdateCommand, "delete": ActionDeleteCommand, + "clone": ActionCloneCommand, }, ) @@ -307,6 +308,88 @@ def run_and_print(self, args, **kwargs): self.print_not_found(resource_id) +class ActionCloneCommand(resource.ContentPackResourceCloneCommand): + source_ref = "source_ref_or_id" + dest_pack = "dest_pack_name" + dest_action = "dest_action_name" + + def __init__(self, resource, *args, **kwargs): + super(ActionCloneCommand, self).__init__(resource, *args, **kwargs) + + args_list = [ + self.source_ref, + self.dest_pack, + self.dest_action, + ] + + for var in args_list: + metavar = self._get_metavar_for_argument(argument=var) + helparg = self._get_help_for_argument(resource=resource, argument=var) + self.parser.add_argument(var, metavar=metavar, help=helparg) + + self.parser.add_argument( + "-f", + "--force", + action="store_true", + dest="force", + help="Overwrite action files on disk if destination exists.", + ) + + @add_auth_token_to_kwargs_from_cli + def run(self, args, **kwargs): + source_ref = getattr(args, self.source_ref, None) + dest_pack = getattr(args, self.dest_pack, None) + dest_action = getattr(args, self.dest_action, None) + dest_ref = "%s.%s" % (dest_pack, dest_action) + self.get_resource_by_ref_or_id(source_ref, **kwargs) + + try: + dest_instance = self.get_resource(dest_ref, **kwargs) + except ResourceNotFoundError: + dest_instance = None + + overwrite = False + + if dest_instance: + user_input = "" + if not args.force: + user_input = input( + "The destination action already exists. Do you want to overwrite? (y/n): " + ) + if args.force or user_input.lower() == "y" or user_input.lower() == "yes": + overwrite = True + else: + print("Action is not cloned.") + return + + return self.manager.clone( + source_ref, + dest_pack, + dest_action, + overwrite=overwrite, + **kwargs, + ) + + def run_and_print(self, args, **kwargs): + try: + instance = self.run(args, **kwargs) + if not instance: + return + self.print_output( + instance, + table.PropertyValueTable, + attributes=["all"], + json=args.json, + yaml=args.yaml, + ) + except ResourceNotFoundError: + source_ref = getattr(args, self.source_ref, None) + self.print_not_found(source_ref) + except Exception as e: + message = six.text_type(e) + print("ERROR: %s" % (message)) + + class ActionRunCommandMixin(object): """ Mixin class which contains utility functions related to action execution. diff --git a/st2client/st2client/commands/resource.py b/st2client/st2client/commands/resource.py index d66e990a93..ef276b59f0 100644 --- a/st2client/st2client/commands/resource.py +++ b/st2client/st2client/commands/resource.py @@ -102,6 +102,7 @@ def __init__( "delete": ResourceDeleteCommand, "enable": ResourceEnableCommand, "disable": ResourceDisableCommand, + "clone": ResourceCloneCommand, } for cmd, cmd_class in cmd_map.items(): if cmd not in commands: @@ -116,6 +117,7 @@ def __init__( self.commands["create"] = commands["create"](*args) self.commands["update"] = commands["update"](*args) self.commands["delete"] = commands["delete"](*args) + self.commands["clone"] = commands["clone"](*args) if has_disable: self.commands["enable"] = commands["enable"](*args) @@ -758,6 +760,28 @@ class ContentPackResourceDeleteCommand(ResourceDeleteCommand): pk_argument_name = "ref_or_id" +class ResourceCloneCommand(ResourceCommand): + def __init__(self, resource, *args, **kwargs): + super(ResourceCloneCommand, self).__init__( + resource, + "clone", + "Clone a new %s." % resource.get_display_name().lower(), + *args, + **kwargs, + ) + + @add_auth_token_to_kwargs_from_cli + def run(self, args, **kwargs): + raise NotImplementedError("clone '%s' is not implemented." % args.parser) + + def run_and_print(self, args, **kwargs): + self.run(args, **kwargs) + + +class ContentPackResourceCloneCommand(ResourceCloneCommand): + pass + + def load_meta_file(file_path): if not os.path.isfile(file_path): raise Exception('File "%s" does not exist.' % file_path) diff --git a/st2client/st2client/models/core.py b/st2client/st2client/models/core.py index 45034b6c6b..f9b6cbf243 100644 --- a/st2client/st2client/models/core.py +++ b/st2client/st2client/models/core.py @@ -391,6 +391,27 @@ def delete_action(self, instance, remove_files, **kwargs): return True + @add_auth_token_to_kwargs_from_env + def clone( + self, + source_ref, + dest_pack, + dest_action, + overwrite, + **kwargs, + ): + url = "/%s/%s/clone" % (self.resource.get_url_path_name(), source_ref) + payload = { + "dest_pack": dest_pack, + "dest_action": dest_action, + "overwrite": overwrite, + } + response = self.client.post(url, payload, **kwargs) + if response.status_code != http_client.OK: + self.handle_error(response) + instance = self.resource.deserialize(parse_api_response(response)) + return instance + @add_auth_token_to_kwargs_from_env def delete_by_id(self, instance_id, **kwargs): url = "/%s/%s" % (self.resource.get_url_path_name(), instance_id) diff --git a/st2client/tests/unit/test_models.py b/st2client/tests/unit/test_models.py index 665942ddcb..7f6f7f9ffa 100644 --- a/st2client/tests/unit/test_models.py +++ b/st2client/tests/unit/test_models.py @@ -446,3 +446,28 @@ def side_effect_checking_verify_parameter_is_not(): self.assertEqual(mock_requests.call_args_list[1][0], call_args) self.assertEqual(mock_requests.call_args_list[1][1], call_kwargs) + + @mock.patch.object( + httpclient.HTTPClient, + "post", + mock.MagicMock( + return_value=base.FakeResponse(json.dumps(base.RESOURCES[0]), 200, "OK") + ), + ) + def test_resource_clone(self): + mgr = models.ResourceManager(base.FakeResource, base.FAKE_ENDPOINT) + source_ref = "spack.saction" + resource = mgr.clone(source_ref, "dpack", "daction", False) + self.assertIsNotNone(resource) + + @mock.patch.object( + httpclient.HTTPClient, + "post", + mock.MagicMock( + return_value=base.FakeResponse("", 500, "INTERNAL SERVER ERROR") + ), + ) + def test_resource_clone_failed(self): + mgr = models.ResourceManager(base.FakeResource, base.FAKE_ENDPOINT) + source_ref = "spack.saction" + self.assertRaises(Exception, mgr.clone, source_ref, "dpack", "daction") diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index c1baf2d135..edc81eb1ff 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -430,6 +430,41 @@ paths: description: Unexpected error schema: $ref: '#/definitions/Error' + /api/v1/actions/{ref_or_id}/clone: + post: + operationId: st2api.controllers.v1.actions:actions_controller.clone + description: | + Clone one action. + parameters: + - name: ref_or_id + in: path + description: Source action reference or id + type: string + required: true + - name: dest_data + in: body + description: Destination action content + schema: + $ref: '#/definitions/ActionCloneRequest' + required: true + x-parameters: + - name: user + in: context + x-as: requester_user + description: User performing the operation. + responses: + '201': + description: Single action being cloned + schema: + $ref: '#/definitions/Action' + examples: + application/json: + ref: 'core.local' + # and stuff + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' /api/v1/actions/views/parameters/{action_id}: get: operationId: st2api.controllers.v1.action_views:parameters_view_controller.get_one @@ -4692,6 +4727,9 @@ definitions: ActionDeleteRequest: allOf: - $ref: '#/definitions/ActionDeleteSchema' + ActionCloneRequest: + allOf: + - $ref: '#/definitions/ActionCloneSchema' ActionParameters: type: object properties: @@ -5372,6 +5410,23 @@ definitions: additionalProperties: False default: [] # additionalProperties: false + ActionCloneSchema: + type: object + properties: + dest_pack: + type: string + description: Destination pack name for cloning + dest_action: + type: string + description: Destination action name for cloning + overwrite: + type: boolean + description: Force clone action if destination already exists + default: false + required: + - dest_pack + - dest_action + additionalProperties: false NotificationPropertySubSchema: type: object properties: diff --git a/st2common/st2common/openapi.yaml.j2 b/st2common/st2common/openapi.yaml.j2 index f2368cd74c..1397c0d201 100644 --- a/st2common/st2common/openapi.yaml.j2 +++ b/st2common/st2common/openapi.yaml.j2 @@ -426,6 +426,41 @@ paths: description: Unexpected error schema: $ref: '#/definitions/Error' + /api/v1/actions/{ref_or_id}/clone: + post: + operationId: st2api.controllers.v1.actions:actions_controller.clone + description: | + Clone one action. + parameters: + - name: ref_or_id + in: path + description: Source action reference or id + type: string + required: true + - name: dest_data + in: body + description: Destination action content + schema: + $ref: '#/definitions/ActionCloneRequest' + required: true + x-parameters: + - name: user + in: context + x-as: requester_user + description: User performing the operation. + responses: + '201': + description: Single action being cloned + schema: + $ref: '#/definitions/Action' + examples: + application/json: + ref: 'core.local' + # and stuff + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' /api/v1/actions/views/parameters/{action_id}: get: operationId: st2api.controllers.v1.action_views:parameters_view_controller.get_one @@ -4688,6 +4723,9 @@ definitions: ActionDeleteRequest: allOf: - $ref: '#/definitions/ActionDeleteSchema' + ActionCloneRequest: + allOf: + - $ref: '#/definitions/ActionCloneSchema' ActionParameters: type: object properties: @@ -5368,6 +5406,23 @@ definitions: additionalProperties: False default: [] # additionalProperties: false + ActionCloneSchema: + type: object + properties: + dest_pack: + type: string + description: Destination pack name for cloning + dest_action: + type: string + description: Destination action name for cloning + overwrite: + type: boolean + description: Force clone action if destination already exists + default: false + required: + - dest_pack + - dest_action + additionalProperties: false NotificationPropertySubSchema: type: object properties: diff --git a/st2common/st2common/router.py b/st2common/st2common/router.py index fa9c002354..36a9476ed6 100644 --- a/st2common/st2common/router.py +++ b/st2common/st2common/router.py @@ -145,6 +145,11 @@ class NotFoundException(Exception): pass +class GenericRequestParam(object): + def __init__(self, **entries): + self.__dict__.update(entries) + + class Request(webob.Request): """ Custom Request implementation which uses our custom and faster json serializer and deserializer. @@ -518,11 +523,6 @@ def __call__(self, req): if content_type == "text/plain": kw[argument_name] = data else: - - class Body(object): - def __init__(self, **entries): - self.__dict__.update(entries) - ref = schema.get("$ref", None) if ref: with self.spec_resolver.resolving(ref) as resolved: @@ -553,10 +553,10 @@ def __init__(self, **entries): ) else: LOG.debug( - "Missing x-api-model definition for %s, using generic Body " + "Missing x-api-model definition for %s, using GenericRequestParam " "model." % (endpoint["operationId"]) ) - model = Body + model = GenericRequestParam instance = self._get_model_instance(model_cls=model, data=data) kw[argument_name] = instance diff --git a/st2common/st2common/services/packs.py b/st2common/st2common/services/packs.py index 74e97e6424..6633fbefde 100644 --- a/st2common/st2common/services/packs.py +++ b/st2common/st2common/services/packs.py @@ -15,6 +15,7 @@ from __future__ import absolute_import +import copy import itertools import os @@ -22,10 +23,13 @@ import six from six.moves import range from oslo_config import cfg +import shutil +import yaml from st2common import log as logging from st2common.content.utils import get_pack_base_path from st2common.exceptions.content import ResourceDiskFilesRemovalError +from st2common.models.db.stormbase import UIDFieldMixin from st2common.persistence.pack import Pack from st2common.util.misc import lowercase_value from st2common.util.jsonify import json_encode @@ -36,6 +40,11 @@ "get_pack_from_index", "search_pack_index", "delete_action_files_from_pack", + "clone_action_files", + "clone_action_db", + "temp_backup_action_files", + "restore_temp_action_files", + "remove_temp_action_files", ] EXCLUDE_FIELDS = ["repo_url", "email"] @@ -290,3 +299,176 @@ def delete_action_files_from_pack(pack_name, entry_point, metadata_file): 'The action metadata file "%s" does not exists on disk.', action_metadata_file_path, ) + + +def _clone_content_to_destination_file(source_file, destination_file): + try: + shutil.copy(src=source_file, dst=destination_file) + except PermissionError: + LOG.error( + 'Unable to copy file to "%s" due to permission error.', + destination_file, + ) + msg = 'Unable to copy file to "%s".' % (destination_file) + raise PermissionError(msg) + except Exception as e: + LOG.error( + 'Unable to copy file to "%s". Exception was "%s".', + destination_file, + e, + ) + msg = ( + 'Unable to copy file to "%s". Please check the logs or ask your ' + "administrator to clone the files manually." % destination_file + ) + raise Exception(msg) + + +def clone_action_files(source_action_db, dest_action_db, dest_pack_base_path): + """ + Prepares the path for entry point and metadata files for source and destination. + Clones the content from source action files to destination action files. + """ + + source_pack = source_action_db["pack"] + source_entry_point = source_action_db["entry_point"] + source_metadata_file = source_action_db["metadata_file"] + source_pack_base_path = get_pack_base_path(pack_name=source_pack) + source_metadata_file_path = os.path.join( + source_pack_base_path, source_metadata_file + ) + dest_metadata_file_name = dest_action_db["metadata_file"] + dest_metadata_file_path = os.path.join(dest_pack_base_path, dest_metadata_file_name) + + # creating actions directory in destination pack if doesn't exist + ac_dir_path = os.path.join(dest_pack_base_path, "actions") + if not os.path.isdir(ac_dir_path): + os.mkdir(path=ac_dir_path) + _clone_content_to_destination_file( + source_file=source_metadata_file_path, destination_file=dest_metadata_file_path + ) + + dest_entry_point = dest_action_db["entry_point"] + dest_runner_type = dest_action_db["runner_type"]["name"] + + if dest_entry_point: + if dest_runner_type in ["orquesta", "action-chain"]: + # creating workflows directory if doesn't exist + wf_dir_path = os.path.join(dest_pack_base_path, "actions", "workflows") + if not os.path.isdir(wf_dir_path): + os.mkdir(path=wf_dir_path) + source_entry_point_file_path = os.path.join( + source_pack_base_path, "actions", source_entry_point + ) + dest_entrypoint_file_path = os.path.join( + dest_pack_base_path, "actions", dest_entry_point + ) + _clone_content_to_destination_file( + source_file=source_entry_point_file_path, + destination_file=dest_entrypoint_file_path, + ) + + with open(dest_metadata_file_path) as df: + doc = yaml.load(df, Loader=yaml.FullLoader) + + doc["name"] = dest_action_db["name"] + if "pack" in doc: + doc["pack"] = dest_action_db["pack"] + doc["entry_point"] = dest_entry_point + + with open(dest_metadata_file_path, "w") as df: + yaml.dump(doc, df, default_flow_style=False, sort_keys=False) + + +def clone_action_db(source_action_db, dest_pack, dest_action): + dest_action_db = copy.deepcopy(source_action_db) + source_runner_type = source_action_db["runner_type"]["name"] + if source_action_db["entry_point"]: + if source_runner_type in ["orquesta", "action-chain"]: + dest_entry_point_file_name = "workflows/%s.yaml" % (dest_action) + else: + old_ext = os.path.splitext(source_action_db["entry_point"])[1] + dest_entry_point_file_name = dest_action + old_ext + else: + dest_entry_point_file_name = "" + dest_action_db["entry_point"] = dest_entry_point_file_name + dest_action_db["metadata_file"] = "actions/%s.yaml" % (dest_action) + dest_action_db["name"] = dest_action + dest_ref = ".".join([dest_pack, dest_action]) + dest_action_db["ref"] = dest_ref + dest_action_db["uid"] = UIDFieldMixin.UID_SEPARATOR.join( + ["action", dest_pack, dest_action] + ) + if "pack" in dest_action_db: + dest_action_db["pack"] = dest_pack + dest_action_db["id"] = None + + return dest_action_db + + +def temp_backup_action_files(pack_base_path, metadata_file, entry_point, temp_sub_dir): + temp_dir_path = "/tmp/%s" % temp_sub_dir + os.mkdir(temp_dir_path) + actions_dir = os.path.join(temp_dir_path, "actions") + os.mkdir(actions_dir) + temp_metadata_file_path = os.path.join(temp_dir_path, metadata_file) + dest_metadata_file_path = os.path.join(pack_base_path, metadata_file) + _clone_content_to_destination_file( + source_file=dest_metadata_file_path, destination_file=temp_metadata_file_path + ) + if entry_point: + entry_point_dir = str(os.path.split(entry_point)[0]) + if entry_point_dir != "": + os.makedirs(os.path.join(actions_dir, entry_point_dir)) + temp_entry_point_file_path = os.path.join(actions_dir, entry_point) + dest_entry_point_file_path = os.path.join( + pack_base_path, "actions", entry_point + ) + _clone_content_to_destination_file( + source_file=dest_entry_point_file_path, + destination_file=temp_entry_point_file_path, + ) + + +def restore_temp_action_files(pack_base_path, metadata_file, entry_point, temp_sub_dir): + temp_dir_path = "/tmp/%s" % temp_sub_dir + temp_metadata_file_path = os.path.join(temp_dir_path, metadata_file) + dest_metadata_file_path = os.path.join(pack_base_path, metadata_file) + _clone_content_to_destination_file( + source_file=temp_metadata_file_path, destination_file=dest_metadata_file_path + ) + if entry_point: + temp_entry_point_file_path = os.path.join(temp_dir_path, "actions", entry_point) + dest_entry_point_file_path = os.path.join( + pack_base_path, "actions", entry_point + ) + _clone_content_to_destination_file( + source_file=temp_entry_point_file_path, + destination_file=dest_entry_point_file_path, + ) + + +def remove_temp_action_files(temp_sub_dir): + temp_dir_path = "/tmp/%s" % temp_sub_dir + if os.path.isdir(temp_dir_path): + try: + shutil.rmtree(temp_dir_path) + except PermissionError: + LOG.error( + 'No permission to delete the "%s" directory', + temp_dir_path, + ) + msg = 'No permission to delete the "%s" directory' % (temp_dir_path) + raise PermissionError(msg) + except Exception as e: + LOG.error( + 'Unable to delete "%s" directory. Exception was "%s"', + temp_dir_path, + e, + ) + msg = ( + 'The temporary directory "%s" could not be removed from disk, please ' + "check the logs or ask your StackStorm administrator to check " + "and delete the temporary directory manually" % (temp_dir_path) + ) + raise Exception(msg) diff --git a/st2common/st2common/validators/api/misc.py b/st2common/st2common/validators/api/misc.py index 215afc5501..290a81a3bd 100644 --- a/st2common/st2common/validators/api/misc.py +++ b/st2common/st2common/validators/api/misc.py @@ -15,9 +15,13 @@ from __future__ import absolute_import from st2common.constants.pack import SYSTEM_PACK_NAME +from st2common.constants.pack import SYSTEM_PACK_NAMES from st2common.exceptions.apivalidation import ValueValidationException -__all__ = ["validate_not_part_of_system_pack"] +__all__ = [ + "validate_not_part_of_system_pack", + "validate_not_part_of_system_pack_by_name", +] def validate_not_part_of_system_pack(resource_db): @@ -37,3 +41,9 @@ def validate_not_part_of_system_pack(resource_db): raise ValueValidationException(msg) return resource_db + + +def validate_not_part_of_system_pack_by_name(pack_name): + if pack_name in SYSTEM_PACK_NAMES: + msg = "Resources belonging to system level packs can't be manipulated" + raise ValueValidationException(msg) diff --git a/st2common/tests/unit/services/test_packs.py b/st2common/tests/unit/services/test_packs.py index 04e68dee51..4186a8d29c 100644 --- a/st2common/tests/unit/services/test_packs.py +++ b/st2common/tests/unit/services/test_packs.py @@ -19,17 +19,186 @@ import os import mock +import shutil import unittest2 +import uuid import st2tests +from st2common.models.db.stormbase import UIDFieldMixin from st2common.services.packs import delete_action_files_from_pack +from st2common.services.packs import clone_action_files +from st2common.services.packs import clone_action_db +from st2common.services.packs import temp_backup_action_files +from st2common.services.packs import restore_temp_action_files +from st2common.services.packs import remove_temp_action_files TEST_PACK = "dummy_pack_1" TEST_PACK_PATH = os.path.join( st2tests.fixturesloader.get_fixtures_packs_base_path(), TEST_PACK ) +TEST_SOURCE_PACK = "core" + +TEST_SOURCE_WORKFLOW_PACK = "orquesta_tests" + +TEST_DEST_PACK = "dummy_pack_23" +TEST_DEST_PACK_PATH = os.path.join( + st2tests.fixturesloader.get_fixtures_packs_base_path(), TEST_DEST_PACK +) + +SOURCE_ACTION_WITH_PYTHON_SCRIPT_RUNNER = { + "description": "Action which injects a new trigger in the system.", + "enabled": True, + "entry_point": "inject_trigger.py", + "metadata_file": "actions/inject_trigger.yaml", + "name": "inject_trigger", + "notify": {}, + "output_schema": {}, + "pack": TEST_SOURCE_PACK, + "parameters": { + "trigger": { + "type": "string", + "description": "Trigger reference (e.g. mypack.my_trigger).", + "required": True, + }, + "payload": {"type": "object", "description": "Trigger payload."}, + "trace_tag": { + "type": "string", + "description": "Optional trace tag.", + "required": False, + }, + }, + "ref": "core.inject_trigger", + "runner_type": {"name": "python-script"}, + "tags": [], + "uid": "action:core:inject_trigger", +} + +SOURCE_ACTION_WITH_SHELL_SCRIPT_RUNNER = { + "description": "This sends an email", + "enabled": True, + "entry_point": "send_mail/send_mail", + "metadata_file": "actions/sendmail.yaml", + "name": "sendmail", + "notify": {}, + "output_schema": {}, + "pack": TEST_SOURCE_PACK, + "parameters": { + "sendmail_binary": { + "description": "Optional path to the sendmail binary. If not provided, it uses a system default one.", + "position": 0, + "required": False, + "type": "string", + "default": "None", + }, + "from": { + "description": "Sender email address.", + "position": 1, + "required": False, + "type": "string", + "default": "stanley", + }, + "to": { + "description": "Recipient email address.", + "position": 2, + "required": True, + "type": "string", + }, + "subject": { + "description": "Subject of the email.", + "position": 3, + "required": True, + "type": "string", + }, + "send_empty_body": { + "description": "Send a message even if the body is empty.", + "position": 4, + "required": False, + "type": "boolean", + "default": True, + }, + "content_type": { + "type": "string", + "description": "Content type of message to be sent without the charset (charset is set to UTF-8 inside the script).", + "default": "text/html", + "position": 5, + }, + "body": { + "description": "Body of the email.", + "position": 6, + "required": True, + "type": "string", + }, + "sudo": {"immutable": True}, + "attachments": { + "description": "Array of attachment file paths, comma-delimited.", + "position": 7, + "required": False, + "type": "string", + }, + }, + "ref": "core.sendmail", + "runner_type": {"name": "local-shell-script"}, + "tags": [], + "uid": "action:core:sendmail", +} + +SOURCE_ACTION_WITH_LOCAL_SHELL_CMD_RUNNER = { + "description": "Action that executes the Linux echo command on the localhost.", + "enabled": True, + "entry_point": "", + "metadata_file": "actions/echo.yaml", + "name": "echo", + "notify": {}, + "output_schema": {}, + "pack": TEST_SOURCE_PACK, + "parameters": { + "message": { + "description": "The message that the command will echo.", + "type": "string", + "required": True, + }, + "cmd": { + "description": "Arbitrary Linux command to be executed on the local host.", + "required": True, + "type": "string", + "default": 'echo "{{message}}"', + "immutable": True, + }, + "kwarg_op": {"immutable": True}, + "sudo": {"default": False, "immutable": True}, + "sudo_password": {"immutable": True}, + }, + "ref": "core.echo", + "runner_type": {"name": "local-shell-cmd"}, + "tags": [], + "uid": "action:core:echo", +} + +# source workflow needed from ``/st2tests/fixtures/packs/`` path. When source workflow +# taken from ``/opt/stackstorm/packs/`` path, related unit tests fail +SOURCE_WORKFLOW = { + "description": "A basic workflow to demonstrate data flow options.", + "enabled": True, + "entry_point": "workflows/data-flow.yaml", + "metadata_file": "actions/data-flow.yaml", + "name": "data-flow", + "notify": {}, + "output_schema": { + "a6": {"type": "string", "required": True}, + "b6": {"type": "string", "required": True}, + "a7": {"type": "string", "required": True}, + "b7": {"type": "string", "required": True, "secret": "********"}, + }, + "pack": TEST_SOURCE_WORKFLOW_PACK, + "parameters": {"a1": {"required": True, "type": "string"}}, + "ref": "orquesta_tests.data-flow", + "runner_type": {"name": "orquesta"}, + "tags": [], + "uid": "action:orquesta_tests:data-flow", +} + class DeleteActionFilesTest(unittest2.TestCase): def test_delete_action_files_from_pack(self): @@ -241,3 +410,432 @@ def test_exception_to_remove_resource_metadata_file(self, remove): # to delete metadata file with self.assertRaisesRegexp(Exception, expected_msg): delete_action_files_from_pack(TEST_PACK, entry_point, metadata_file) + + +class CloneActionDBAndFilesTestCase(unittest2.TestCase): + @classmethod + def setUpClass(cls): + action_files_path = os.path.join(TEST_DEST_PACK_PATH, "actions") + workflow_files_path = os.path.join(action_files_path, "workflows") + if not os.path.isdir(action_files_path): + os.mkdir(action_files_path) + if not os.path.isdir(workflow_files_path): + os.mkdir(workflow_files_path) + + @classmethod + def tearDownClass(cls): + action_files_path = os.path.join(TEST_DEST_PACK_PATH, "actions") + workflow_files_path = os.path.join(action_files_path, "workflows") + if not os.path.isdir(action_files_path): + os.mkdir(action_files_path) + if not os.path.isdir(workflow_files_path): + os.mkdir(workflow_files_path) + for file in os.listdir(action_files_path): + if os.path.isfile(os.path.join(action_files_path, file)): + os.remove(os.path.join(action_files_path, file)) + for file in os.listdir(workflow_files_path): + if os.path.isfile(os.path.join(workflow_files_path, file)): + os.remove(os.path.join(workflow_files_path, file)) + + def test_clone_action_db(self): + CLONE_ACTION_1 = clone_action_db( + SOURCE_ACTION_WITH_PYTHON_SCRIPT_RUNNER, TEST_DEST_PACK, "clone_action_1" + ) + exptected_uid = UIDFieldMixin.UID_SEPARATOR.join( + ["action", TEST_DEST_PACK, "clone_action_1"] + ) + actual_uid = CLONE_ACTION_1["uid"] + self.assertEqual(actual_uid, exptected_uid) + exptected_parameters = SOURCE_ACTION_WITH_PYTHON_SCRIPT_RUNNER["parameters"] + actual_parameters = CLONE_ACTION_1["parameters"] + self.assertDictEqual(actual_parameters, exptected_parameters) + + def test_clone_files_for_python_script_runner_action(self): + CLONE_ACTION_1 = clone_action_db( + SOURCE_ACTION_WITH_PYTHON_SCRIPT_RUNNER, TEST_DEST_PACK, "clone_action_1" + ) + clone_action_files( + SOURCE_ACTION_WITH_PYTHON_SCRIPT_RUNNER, CLONE_ACTION_1, TEST_DEST_PACK_PATH + ) + cloned_action_metadata_file_path = os.path.join( + TEST_DEST_PACK_PATH, "actions", "clone_action_1.yaml" + ) + cloned_action_entry_point_file_path = os.path.join( + TEST_DEST_PACK_PATH, "actions", "clone_action_1.py" + ) + self.assertTrue(os.path.exists(cloned_action_metadata_file_path)) + self.assertTrue(os.path.exists(cloned_action_entry_point_file_path)) + + def test_clone_files_for_shell_script_runner_action(self): + CLONE_ACTION_2 = clone_action_db( + SOURCE_ACTION_WITH_SHELL_SCRIPT_RUNNER, TEST_DEST_PACK, "clone_action_2" + ) + clone_action_files( + SOURCE_ACTION_WITH_SHELL_SCRIPT_RUNNER, CLONE_ACTION_2, TEST_DEST_PACK_PATH + ) + cloned_action_metadata_file_path = os.path.join( + TEST_DEST_PACK_PATH, "actions", "clone_action_2.yaml" + ) + cloned_action_entry_point_file_path = os.path.join( + TEST_DEST_PACK_PATH, "actions", "clone_action_2" + ) + self.assertTrue(os.path.exists(cloned_action_metadata_file_path)) + self.assertTrue(os.path.exists(cloned_action_entry_point_file_path)) + + def test_clone_files_for_local_shell_cmd_runner_action(self): + CLONE_ACTION_3 = clone_action_db( + SOURCE_ACTION_WITH_LOCAL_SHELL_CMD_RUNNER, TEST_DEST_PACK, "clone_action_3" + ) + clone_action_files( + SOURCE_ACTION_WITH_LOCAL_SHELL_CMD_RUNNER, + CLONE_ACTION_3, + TEST_DEST_PACK_PATH, + ) + cloned_action_metadata_file_path = os.path.join( + TEST_DEST_PACK_PATH, "actions", "clone_action_3.yaml" + ) + self.assertTrue(os.path.exists(cloned_action_metadata_file_path)) + + def test_clone_files_for_workflow_action(self): + CLONE_WORKFLOW = clone_action_db( + SOURCE_WORKFLOW, TEST_DEST_PACK, "clone_workflow" + ) + clone_action_files(SOURCE_WORKFLOW, CLONE_WORKFLOW, TEST_DEST_PACK_PATH) + cloned_workflow_metadata_file_path = os.path.join( + TEST_DEST_PACK_PATH, "actions", "clone_workflow.yaml" + ) + cloned_workflow_entry_point_file_path = os.path.join( + TEST_DEST_PACK_PATH, "actions", "workflows", "clone_workflow.yaml" + ) + self.assertTrue(os.path.exists(cloned_workflow_metadata_file_path)) + self.assertTrue(os.path.exists(cloned_workflow_entry_point_file_path)) + + @mock.patch("shutil.copy") + def test_permission_error_to_write_in_destination_file(self, mock_copy): + mock_copy.side_effect = PermissionError("No permission to write in file") + cloned_action_metadata_file_path = os.path.join( + TEST_DEST_PACK_PATH, "actions", "clone_action_4.yaml" + ) + expected_msg = 'Unable to copy file to "%s".' % ( + cloned_action_metadata_file_path + ) + CLONE_ACTION_4 = clone_action_db( + SOURCE_ACTION_WITH_SHELL_SCRIPT_RUNNER, TEST_DEST_PACK, "clone_action_4" + ) + with self.assertRaisesRegexp(PermissionError, expected_msg): + clone_action_files( + SOURCE_ACTION_WITH_SHELL_SCRIPT_RUNNER, + CLONE_ACTION_4, + TEST_DEST_PACK_PATH, + ) + + @mock.patch("shutil.copy") + def test_exceptions_to_write_in_destination_file(self, mock_copy): + mock_copy.side_effect = Exception( + "Exception encoutntered during writing in destination action file" + ) + CLONE_ACTION_5 = clone_action_db( + SOURCE_ACTION_WITH_LOCAL_SHELL_CMD_RUNNER, TEST_DEST_PACK, "clone_action_5" + ) + cloned_action_metadata_file_path = os.path.join( + TEST_DEST_PACK_PATH, "actions", "clone_action_5.yaml" + ) + expected_msg = ( + 'Unable to copy file to "%s". Please check the logs or ask your ' + "administrator to clone the files manually." + % cloned_action_metadata_file_path + ) + with self.assertRaisesRegexp(Exception, expected_msg): + clone_action_files( + SOURCE_ACTION_WITH_LOCAL_SHELL_CMD_RUNNER, + CLONE_ACTION_5, + TEST_DEST_PACK_PATH, + ) + + def test_actions_directory_created_if_does_not_exist(self): + action_dir_path = os.path.join(TEST_DEST_PACK_PATH, "actions") + # removing actions directory and asserting it doesn't exist + shutil.rmtree(action_dir_path) + self.assertFalse(os.path.exists(action_dir_path)) + CLONE_ACTION_6 = clone_action_db( + SOURCE_ACTION_WITH_LOCAL_SHELL_CMD_RUNNER, TEST_DEST_PACK, "clone_action_6" + ) + clone_action_files( + SOURCE_ACTION_WITH_LOCAL_SHELL_CMD_RUNNER, + CLONE_ACTION_6, + TEST_DEST_PACK_PATH, + ) + # workflows directory created and asserting it exists + self.assertTrue(os.path.exists(action_dir_path)) + wf_dir_path = os.path.join(action_dir_path, "workflows") + if not os.path.isdir(wf_dir_path): + os.mkdir(wf_dir_path) + + def test_workflows_directory_created_if_does_not_exist(self): + action_dir_path = os.path.join(TEST_DEST_PACK_PATH, "actions") + workflows_dir_path = os.path.join(TEST_DEST_PACK_PATH, "actions", "workflows") + # removing workflows directory and asserting it doesn't exist + shutil.rmtree(workflows_dir_path) + self.assertFalse(os.path.exists(workflows_dir_path)) + self.assertTrue(os.path.exists(action_dir_path)) + CLONE_WORKFLOW = clone_action_db( + SOURCE_WORKFLOW, TEST_DEST_PACK, "clone_workflow" + ) + clone_action_files(SOURCE_WORKFLOW, CLONE_WORKFLOW, TEST_DEST_PACK_PATH) + # workflows directory created and asserting it exists + self.assertTrue(os.path.exists(workflows_dir_path)) + + +class CloneActionFilesBackupTestCase(unittest2.TestCase): + @classmethod + def tearDownClass(cls): + action_files_path = os.path.join(TEST_DEST_PACK_PATH, "actions") + workflow_files_path = os.path.join(action_files_path, "workflows") + for file in os.listdir(action_files_path): + if os.path.isfile(os.path.join(action_files_path, file)): + os.remove(os.path.join(action_files_path, file)) + for file in os.listdir(workflow_files_path): + if os.path.isfile(os.path.join(workflow_files_path, file)): + os.remove(os.path.join(workflow_files_path, file)) + + def test_temp_backup_restore_remove_action_files(self): + CLONE_ACTION_1 = clone_action_db( + SOURCE_ACTION_WITH_PYTHON_SCRIPT_RUNNER, TEST_DEST_PACK, "clone_action_1" + ) + clone_action_files( + SOURCE_ACTION_WITH_PYTHON_SCRIPT_RUNNER, CLONE_ACTION_1, TEST_DEST_PACK_PATH + ) + dest_action_metadata_file = CLONE_ACTION_1["metadata_file"] + dest_action_entry_point_file = CLONE_ACTION_1["entry_point"] + temp_sub_dir = str(uuid.uuid4()) + + # creating backup of dest action files at /tmp/ + temp_backup_action_files( + TEST_DEST_PACK_PATH, + dest_action_metadata_file, + dest_action_entry_point_file, + temp_sub_dir, + ) + temp_dir_path = "/tmp/%s" % temp_sub_dir + self.assertTrue(os.path.isdir(temp_dir_path)) + temp_metadata_file_path = os.path.join(temp_dir_path, dest_action_metadata_file) + temp_entry_point_file_path = os.path.join( + temp_dir_path, "actions", dest_action_entry_point_file + ) + # asserting backup files exists + self.assertTrue(os.path.exists(temp_metadata_file_path)) + self.assertTrue(os.path.exists(temp_entry_point_file_path)) + + # removing destination action files + dest_action_files_path = os.path.join(TEST_DEST_PACK_PATH, "actions") + for file in os.listdir(dest_action_files_path): + if os.path.isfile(os.path.join(dest_action_files_path, file)): + os.remove(os.path.join(dest_action_files_path, file)) + dest_action_metadata_file_path = os.path.join( + TEST_DEST_PACK_PATH, dest_action_metadata_file + ) + dest_action_entry_point_file_path = os.path.join( + TEST_DEST_PACK_PATH, "actions", dest_action_entry_point_file + ) + # asserting destination action files doesn't exist + self.assertFalse(os.path.isfile(dest_action_metadata_file_path)) + self.assertFalse(os.path.isfile(dest_action_entry_point_file_path)) + + # restoring temp backed action files to destination + restore_temp_action_files( + TEST_DEST_PACK_PATH, + dest_action_metadata_file, + dest_action_entry_point_file, + temp_sub_dir, + ) + # asserting action files restored at destination + self.assertTrue(os.path.isfile(dest_action_metadata_file_path)) + self.assertTrue(os.path.isfile(dest_action_entry_point_file_path)) + # asserting temp_dir and backed action files exits + self.assertTrue(os.path.isdir(temp_dir_path)) + self.assertTrue(os.path.exists(temp_metadata_file_path)) + self.assertTrue(os.path.exists(temp_entry_point_file_path)) + + # removing temp_dir and backed action files + remove_temp_action_files(temp_sub_dir) + # asserting temp_dir and backed action files doesn't exit + self.assertFalse(os.path.isdir(temp_dir_path)) + self.assertFalse(os.path.exists(temp_metadata_file_path)) + self.assertFalse(os.path.exists(temp_entry_point_file_path)) + + def test_exception_remove_temp_action_files(self): + CLONE_ACTION_4 = clone_action_db( + SOURCE_ACTION_WITH_PYTHON_SCRIPT_RUNNER, TEST_DEST_PACK, "clone_action_4" + ) + clone_action_files( + SOURCE_ACTION_WITH_PYTHON_SCRIPT_RUNNER, CLONE_ACTION_4, TEST_DEST_PACK_PATH + ) + dest_action_metadata_file = CLONE_ACTION_4["metadata_file"] + dest_action_entry_point_file = CLONE_ACTION_4["entry_point"] + temp_sub_dir = str(uuid.uuid4()) + temp_backup_action_files( + TEST_DEST_PACK_PATH, + dest_action_metadata_file, + dest_action_entry_point_file, + temp_sub_dir, + ) + temp_dir_path = "/tmp/%s" % temp_sub_dir + self.assertTrue(os.path.isdir(temp_dir_path)) + expected_msg = ( + 'The temporary directory "%s" could not be removed from disk, please check the logs ' + "or ask your StackStorm administrator to check and delete the temporary directory " + "manually" % temp_dir_path + ) + with mock.patch("shutil.rmtree") as mock_rmdir: + mock_rmdir.side_effect = Exception + with self.assertRaisesRegexp(Exception, expected_msg): + remove_temp_action_files(temp_sub_dir) + + remove_temp_action_files(temp_sub_dir) + + def test_permission_error_remove_temp_action_files(self): + CLONE_ACTION_5 = clone_action_db( + SOURCE_ACTION_WITH_PYTHON_SCRIPT_RUNNER, TEST_DEST_PACK, "clone_action_5" + ) + clone_action_files( + SOURCE_ACTION_WITH_PYTHON_SCRIPT_RUNNER, CLONE_ACTION_5, TEST_DEST_PACK_PATH + ) + dest_action_metadata_file = CLONE_ACTION_5["metadata_file"] + dest_action_entry_point_file = CLONE_ACTION_5["entry_point"] + temp_sub_dir = str(uuid.uuid4()) + temp_backup_action_files( + TEST_DEST_PACK_PATH, + dest_action_metadata_file, + dest_action_entry_point_file, + temp_sub_dir, + ) + temp_dir_path = "/tmp/%s" % temp_sub_dir + self.assertTrue(os.path.isdir(temp_dir_path)) + expected_msg = 'No permission to delete the "%s" directory' % temp_dir_path + with mock.patch("shutil.rmtree") as mock_rmdir: + mock_rmdir.side_effect = PermissionError + with self.assertRaisesRegexp(PermissionError, expected_msg): + remove_temp_action_files(temp_sub_dir) + + remove_temp_action_files(temp_sub_dir) + + def test_exception_temp_backup_action_files(self): + CLONE_ACTION_6 = clone_action_db( + SOURCE_ACTION_WITH_SHELL_SCRIPT_RUNNER, TEST_DEST_PACK, "clone_action_6" + ) + clone_action_files( + SOURCE_ACTION_WITH_SHELL_SCRIPT_RUNNER, CLONE_ACTION_6, TEST_DEST_PACK_PATH + ) + dest_action_metadata_file = CLONE_ACTION_6["metadata_file"] + dest_action_entry_point_file = CLONE_ACTION_6["entry_point"] + temp_sub_dir = str(uuid.uuid4()) + temp_dir_path = "/tmp/%s" % temp_sub_dir + tmp_action_metadata_file_path = os.path.join( + temp_dir_path, dest_action_metadata_file + ) + expected_msg = ( + 'Unable to copy file to "%s". Please check the logs or ask your ' + "administrator to clone the files manually." % tmp_action_metadata_file_path + ) + with mock.patch("shutil.copy") as mock_copy: + mock_copy.side_effect = Exception + with self.assertRaisesRegexp(Exception, expected_msg): + temp_backup_action_files( + TEST_DEST_PACK_PATH, + dest_action_metadata_file, + dest_action_entry_point_file, + temp_sub_dir, + ) + + remove_temp_action_files(temp_sub_dir) + + def test_permission_error_temp_backup_action_files(self): + CLONE_ACTION_7 = clone_action_db( + SOURCE_ACTION_WITH_SHELL_SCRIPT_RUNNER, TEST_DEST_PACK, "clone_action_7" + ) + clone_action_files( + SOURCE_ACTION_WITH_SHELL_SCRIPT_RUNNER, CLONE_ACTION_7, TEST_DEST_PACK_PATH + ) + dest_action_metadata_file = CLONE_ACTION_7["metadata_file"] + dest_action_entry_point_file = CLONE_ACTION_7["entry_point"] + temp_sub_dir = str(uuid.uuid4()) + temp_dir_path = "/tmp/%s" % temp_sub_dir + tmp_action_metadata_file_path = os.path.join( + temp_dir_path, dest_action_metadata_file + ) + expected_msg = 'Unable to copy file to "%s".' % tmp_action_metadata_file_path + with mock.patch("shutil.copy") as mock_copy: + mock_copy.side_effect = PermissionError + with self.assertRaisesRegexp(PermissionError, expected_msg): + temp_backup_action_files( + TEST_DEST_PACK_PATH, + dest_action_metadata_file, + dest_action_entry_point_file, + temp_sub_dir, + ) + + remove_temp_action_files(temp_sub_dir) + + def test_exception_restore_temp_action_files(self): + CLONE_ACTION_8 = clone_action_db( + SOURCE_ACTION_WITH_SHELL_SCRIPT_RUNNER, TEST_DEST_PACK, "clone_action_8" + ) + clone_action_files( + SOURCE_ACTION_WITH_SHELL_SCRIPT_RUNNER, CLONE_ACTION_8, TEST_DEST_PACK_PATH + ) + dest_action_metadata_file = CLONE_ACTION_8["metadata_file"] + dest_action_entry_point_file = CLONE_ACTION_8["entry_point"] + dest_action_files_path = os.path.join(TEST_DEST_PACK_PATH, "actions") + for file in os.listdir(dest_action_files_path): + if os.path.isfile(os.path.join(dest_action_files_path, file)): + os.remove(os.path.join(dest_action_files_path, file)) + temp_sub_dir = str(uuid.uuid4()) + dest_action_metadata_file_path = os.path.join( + TEST_DEST_PACK_PATH, dest_action_metadata_file + ) + expected_msg = ( + 'Unable to copy file to "%s". Please check the logs or ask your ' + "administrator to clone the files manually." + % dest_action_metadata_file_path + ) + with mock.patch("shutil.copy") as mock_copy: + mock_copy.side_effect = Exception + with self.assertRaisesRegexp(Exception, expected_msg): + restore_temp_action_files( + TEST_DEST_PACK_PATH, + dest_action_metadata_file, + dest_action_entry_point_file, + temp_sub_dir, + ) + + remove_temp_action_files(temp_sub_dir) + + def test_permission_error_restore_temp_action_files(self): + CLONE_ACTION_9 = clone_action_db( + SOURCE_ACTION_WITH_SHELL_SCRIPT_RUNNER, TEST_DEST_PACK, "clone_action_9" + ) + clone_action_files( + SOURCE_ACTION_WITH_SHELL_SCRIPT_RUNNER, CLONE_ACTION_9, TEST_DEST_PACK_PATH + ) + dest_action_metadata_file = CLONE_ACTION_9["metadata_file"] + dest_action_entry_point_file = CLONE_ACTION_9["entry_point"] + dest_action_files_path = os.path.join(TEST_DEST_PACK_PATH, "actions") + for file in os.listdir(dest_action_files_path): + if os.path.isfile(os.path.join(dest_action_files_path, file)): + os.remove(os.path.join(dest_action_files_path, file)) + temp_sub_dir = str(uuid.uuid4()) + dest_action_metadata_file_path = os.path.join( + TEST_DEST_PACK_PATH, dest_action_metadata_file + ) + expected_msg = 'Unable to copy file to "%s".' % dest_action_metadata_file_path + with mock.patch("shutil.copy") as mock_copy: + mock_copy.side_effect = PermissionError + with self.assertRaisesRegexp(PermissionError, expected_msg): + restore_temp_action_files( + TEST_DEST_PACK_PATH, + dest_action_metadata_file, + dest_action_entry_point_file, + temp_sub_dir, + ) + + remove_temp_action_files(temp_sub_dir) diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_23/actions/workflows/__init__.py b/st2tests/st2tests/fixtures/packs/dummy_pack_23/actions/workflows/__init__.py new file mode 100644 index 0000000000..30c83ff44f --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_23/actions/workflows/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 The StackStorm Authors. +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.