diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b423a86e0c..7b93e1c2b4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -30,6 +30,10 @@ in development * Fix bug in triggers emitted on key value pair changes and sensor spawn/exit. When dispatching those triggers, the reference used didn't contain the pack names which meant it was invalid and lookups in the rules engine would fail. (bug-fix) +* Allow user to include files which are written on disk inside the action create API payload. + (new feature) +* Allow user to retrieve content of a file inside a pack by using the new + ``/packs/views/files/`` API endpoint. (new feature) 0.12.1 - July 31, 2015 ---------------------- diff --git a/st2actions/tests/unit/test_runner_container_service.py b/st2actions/tests/unit/test_runner_container_service.py index 0f97f09468..c1ab1807c5 100644 --- a/st2actions/tests/unit/test_runner_container_service.py +++ b/st2actions/tests/unit/test_runner_container_service.py @@ -50,8 +50,9 @@ def test_get_entry_point_absolute_path(self): service = RunnerContainerService() orig_path = cfg.CONF.content.system_packs_base_path cfg.CONF.content.system_packs_base_path = '/tests/packs' - acutal_path = service.get_entry_point_abs_path(pack='foo', entry_point='/foo/bar.py') - self.assertEqual(acutal_path, '/foo/bar.py', 'Entry point path doesn\'t match.') + acutal_path = service.get_entry_point_abs_path(pack='foo', + entry_point='/tests/packs/foo/bar.py') + self.assertEqual(acutal_path, '/tests/packs/foo/bar.py', 'Entry point path doesn\'t match.') cfg.CONF.content.system_packs_base_path = orig_path def test_get_entry_point_absolute_path_empty(self): @@ -86,7 +87,8 @@ def test_get_action_libs_abs_path(self): self.assertEqual(acutal_path, expected_path, 'Action libs path doesn\'t match.') # entry point absolute. - acutal_path = service.get_action_libs_abs_path(pack='foo', entry_point='/tmp/foo.py') - expected_path = os.path.join('/tmp', ACTION_LIBS_DIR) + acutal_path = service.get_action_libs_abs_path(pack='foo', + entry_point='/tests/packs/foo/tmp/foo.py') + expected_path = os.path.join('/tests/packs/foo/tmp', ACTION_LIBS_DIR) self.assertEqual(acutal_path, expected_path, 'Action libs path doesn\'t match.') cfg.CONF.content.system_packs_base_path = orig_path diff --git a/st2api/st2api/controllers/resource.py b/st2api/st2api/controllers/resource.py index f0d8174701..d3b7af593f 100644 --- a/st2api/st2api/controllers/resource.py +++ b/st2api/st2api/controllers/resource.py @@ -181,7 +181,7 @@ def _get_one_by_name_or_id(self, name_or_id, exclude_fields=None): from_model_kwargs = self._get_from_model_kwargs_for_request(request=pecan.request) result = self.model.from_model(instance, **from_model_kwargs) - LOG.debug('GET %s with name_or_odid=%s, client_result=%s', pecan.request.path, id, result) + LOG.debug('GET %s with name_or_id=%s, client_result=%s', pecan.request.path, id, result) return result @@ -235,11 +235,11 @@ def get_one(self, ref_or_id): def get_all(self, **kwargs): return self._get_all(**kwargs) - def _get_one(self, ref_or_id): + def _get_one(self, ref_or_id, exclude_fields=None): LOG.info('GET %s with ref_or_id=%s', pecan.request.path, ref_or_id) try: - instance = self._get_by_ref_or_id(ref_or_id=ref_or_id) + instance = self._get_by_ref_or_id(ref_or_id=ref_or_id, exclude_fields=exclude_fields) except Exception as e: LOG.exception(e.message) pecan.abort(http_client.NOT_FOUND, e.message) @@ -268,7 +268,7 @@ def _get_all(self, **kwargs): return result - def _get_by_ref_or_id(self, ref_or_id): + def _get_by_ref_or_id(self, ref_or_id, exclude_fields=None): """ Retrieve resource object by an id of a reference. @@ -283,9 +283,9 @@ def _get_by_ref_or_id(self, ref_or_id): is_reference = False if is_reference: - resource_db = self._get_by_ref(resource_ref=ref_or_id) + resource_db = self._get_by_ref(resource_ref=ref_or_id, exclude_fields=exclude_fields) else: - resource_db = self._get_by_id(resource_id=ref_or_id) + resource_db = self._get_by_id(resource_id=ref_or_id, exclude_fields=exclude_fields) if not resource_db: msg = 'Resource with a reference or id "%s" not found' % (ref_or_id) @@ -293,21 +293,14 @@ def _get_by_ref_or_id(self, ref_or_id): return resource_db - def _get_by_id(self, resource_id): - try: - resource_db = self.access.get_by_id(resource_id) - except Exception: - resource_db = None - - return resource_db - - def _get_by_ref(self, resource_ref): + def _get_by_ref(self, resource_ref, exclude_fields=None): try: ref = ResourceReference.from_string_reference(ref=resource_ref) except Exception: return None - resource_db = self.access.query(name=ref.name, pack=ref.pack).first() + resource_db = self.access.query(name=ref.name, pack=ref.pack, + exclude_fields=exclude_fields).first() return resource_db def _get_filters(self, **kwargs): diff --git a/st2api/st2api/controllers/v1/actions.py b/st2api/st2api/controllers/v1/actions.py index e95c26a88a..845dfa7b7b 100644 --- a/st2api/st2api/controllers/v1/actions.py +++ b/st2api/st2api/controllers/v1/actions.py @@ -13,10 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from mongoengine import ValidationError +import os +import os.path -from pecan import abort import six +from pecan import abort +from mongoengine import ValidationError # TODO: Encapsulate mongoengine errors in our persistence layer. Exceptions # that bubble up to this layer should be core Python exceptions or @@ -26,11 +28,19 @@ from st2api.controllers.v1.actionviews import ActionViewsController from st2common import log as logging from st2common.constants.pack import DEFAULT_PACK_NAME +from st2common.constants.triggers import ACTION_FILE_WRITTEN_TRIGGER from st2common.exceptions.apivalidation import ValueValidationException from st2common.models.api.base import jsexpose from st2common.persistence.action import Action from st2common.models.api.action import ActionAPI +from st2common.models.api.action import ActionCreateAPI +from st2common.persistence.pack import Pack from st2common.validators.api.misc import validate_not_part_of_system_pack +from st2common.content.utils import get_pack_base_path +from st2common.content.utils import get_pack_resource_file_abs_path +from st2common.content.utils import get_relative_path_to_pack +from st2common.transport.reactor import TriggerDispatcher +from st2common.util.system_info import get_host_info import st2common.validators.api.action as action_validator http_client = six.moves.http_client @@ -58,7 +68,11 @@ class ActionsController(resource.ContentPackResourceController): include_reference = True - @jsexpose(body_cls=ActionAPI, status_code=http_client.CREATED) + def __init__(self, *args, **kwargs): + super(ActionsController, self).__init__(*args, **kwargs) + self._trigger_dispatcher = TriggerDispatcher(LOG) + + @jsexpose(body_cls=ActionCreateAPI, status_code=http_client.CREATED) def post(self, action): """ Create a new action. @@ -73,19 +87,32 @@ def post(self, action): validate_not_part_of_system_pack(action) action_validator.validate_action(action) + # Write pack data files to disk (if any are provided) + data_files = getattr(action, 'data_files', []) + written_data_files = [] + if data_files: + written_data_files = self._handle_data_files(pack_name=action.pack, + data_files=data_files) + action_model = ActionAPI.to_model(action) LOG.debug('/actions/ POST verified ActionAPI object=%s', action) action_db = Action.add_or_update(action_model) LOG.debug('/actions/ POST saved ActionDB object=%s', action_db) - extra = {'action_db': action_db} + # Dispatch an internal trigger for each written data file. This way user + # automate comitting this files to git using StackStorm rule + if written_data_files: + self._dispatch_trigger_for_written_data_files(action_db=action_db, + written_data_files=written_data_files) + + extra = {'acion_db': action_db} LOG.audit('Action created. Action.id=%s' % (action_db.id), extra=extra) action_api = ActionAPI.from_model(action_db) return action_api - @jsexpose(arg_types=[str], body_cls=ActionAPI) + @jsexpose(arg_types=[str], body_cls=ActionCreateAPI) def put(self, action_ref_or_id, action): action_db = self._get_by_ref_or_id(ref_or_id=action_ref_or_id) action_id = action_db.id @@ -97,6 +124,13 @@ def put(self, action_ref_or_id, action): validate_not_part_of_system_pack(action) action_validator.validate_action(action) + # Write pack data files to disk (if any are provided) + data_files = getattr(action, 'data_files', []) + written_data_files = [] + if data_files: + written_data_files = self._handle_data_files(pack_name=action.pack, + data_files=data_files) + try: action_db = ActionAPI.to_model(action) action_db.id = action_id @@ -106,6 +140,12 @@ def put(self, action_ref_or_id, action): abort(http_client.BAD_REQUEST, str(e)) return + # Dispatch an internal trigger for each written data file. This way user + # automate comitting this files to git using StackStorm rule + if written_data_files: + self._dispatch_trigger_for_written_data_files(action_db=action_db, + written_data_files=written_data_files) + action_api = ActionAPI.from_model(action_db) LOG.debug('PUT /actions/ client_result=%s', action_api) @@ -143,3 +183,90 @@ def delete(self, action_ref_or_id): extra = {'action_db': action_db} LOG.audit('Action deleted. Action.id=%s' % (action_db.id), extra=extra) return None + + def _handle_data_files(self, pack_name, data_files): + """ + Method for handling action data files. + + This method performs two tasks: + + 1. Writes files to disk + 2. Updates affected PackDB model + """ + # Write files to disk + written_file_paths = self._write_data_files_to_disk(pack_name=pack_name, + data_files=data_files) + + # Update affected PackDB model (update a list of files) + # Update PackDB + self._update_pack_model(pack_name=pack_name, data_files=data_files, + written_file_paths=written_file_paths) + + return written_file_paths + + def _write_data_files_to_disk(self, pack_name, data_files): + """ + Write files to disk. + """ + written_file_paths = [] + + for data_file in data_files: + file_path = data_file['file_path'] + content = data_file['content'] + + file_path = get_pack_resource_file_abs_path(pack_name=pack_name, + resource_type='action', + file_path=file_path) + + LOG.debug('Writing data file "%s" to "%s"' % (str(data_file), file_path)) + self._write_data_file(pack_name=pack_name, file_path=file_path, content=content) + written_file_paths.append(file_path) + + return written_file_paths + + def _update_pack_model(self, pack_name, data_files, written_file_paths): + """ + Update PackDB models (update files list). + """ + file_paths = [] # A list of paths relative to the pack directory for new files + for file_path in written_file_paths: + file_path = get_relative_path_to_pack(pack_name=pack_name, file_path=file_path) + file_paths.append(file_path) + + pack_db = Pack.get_by_ref(pack_name) + pack_db.files = set(pack_db.files) + pack_db.files.update(set(file_paths)) + pack_db.files = list(pack_db.files) + pack_db = Pack.add_or_update(pack_db) + + return pack_db + + def _write_data_file(self, pack_name, file_path, content): + """ + Write data file on disk. + """ + # Throw if pack directory doesn't exist + pack_base_path = get_pack_base_path(pack_name=pack_name) + if not os.path.isdir(pack_base_path): + raise ValueError('Directory for pack "%s" doesn\'t exist' % (pack_name)) + + # Create pack sub-directory tree if it doesn't exist + directory = os.path.dirname(file_path) + + if not os.path.isdir(directory): + os.makedirs(directory) + + with open(file_path, 'w') as fp: + fp.write(content) + + def _dispatch_trigger_for_written_data_files(self, action_db, written_data_files): + trigger = ACTION_FILE_WRITTEN_TRIGGER['name'] + host_info = get_host_info() + + for file_path in written_data_files: + payload = { + 'ref': action_db.ref, + 'file_path': file_path, + 'host_info': host_info + } + self._trigger_dispatcher.dispatch(trigger=trigger, payload=payload) diff --git a/st2api/st2api/controllers/v1/packs.py b/st2api/st2api/controllers/v1/packs.py index 947dcff9b2..c233840170 100644 --- a/st2api/st2api/controllers/v1/packs.py +++ b/st2api/st2api/controllers/v1/packs.py @@ -15,6 +15,7 @@ from st2common.models.api.base import jsexpose from st2api.controllers.resource import ResourceController +from st2api.controllers.v1.packviews import PackViewsController from st2common.models.api.pack import PackAPI from st2common.persistence.pack import Pack @@ -35,6 +36,9 @@ class PacksController(ResourceController): 'sort': ['ref'] } + # Nested controllers + views = PackViewsController() + @jsexpose(arg_types=[str]) def get_one(self, name_or_id): return self._get_one_by_name_or_id(name_or_id=name_or_id) diff --git a/st2api/st2api/controllers/v1/packviews.py b/st2api/st2api/controllers/v1/packviews.py new file mode 100644 index 0000000000..d680fbe2f3 --- /dev/null +++ b/st2api/st2api/controllers/v1/packviews.py @@ -0,0 +1,119 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import six +from pecan import abort +from pecan.rest import RestController + +from st2api.controllers import resource +from st2common.exceptions.db import StackStormDBObjectNotFoundError +from st2common import log as logging +from st2common.models.api.base import jsexpose +from st2common.models.api.pack import PackAPI +from st2common.persistence.pack import Pack +from st2common.content.utils import get_pack_file_abs_path + +http_client = six.moves.http_client + +LOG = logging.getLogger(__name__) + + +class BaseFileController(resource.ResourceController): + model = PackAPI + access = Pack + + supported_filters = {} + + @jsexpose() + def get_all(self, **kwargs): + return abort(404) + + def _get_file_content(self, file_path): + with open(file_path, 'r') as fp: + content = fp.read() + + return content + + +class FilesController(BaseFileController): + """ + Controller which allows user to retrieve content of all the files inside the pack. + """ + + @jsexpose(arg_types=[str], status_code=http_client.OK) + def get_one(self, name_or_id): + """ + Outputs the content of all the files inside the pack. + + Handles requests: + GET /packs/views/files/ + """ + pack_db = self._get_by_name_or_id(name_or_id=name_or_id) + pack_name = pack_db.name + pack_files = pack_db.files + + result = [] + for file_path in pack_files: + normalized_file_path = get_pack_file_abs_path(pack_name=pack_name, file_path=file_path) + + if not normalized_file_path or not os.path.isfile(normalized_file_path): + # Ignore references to files which don't exist on disk + continue + + content = self._get_file_content(file_path=normalized_file_path) + item = { + 'file_path': file_path, + 'content': content + } + result.append(item) + + return result + + +class FileController(BaseFileController): + """ + Controller which allows user to retrieve content of a specific file in a pack. + """ + @jsexpose(content_type='text/plain', status_code=http_client.OK) + def get_one(self, name_or_id, *file_path_components): + """ + Outputs the content of all the files inside the pack. + + Handles requests: + GET /packs/views/files// + """ + pack_db = self._get_by_name_or_id(name_or_id=name_or_id) + + if not file_path_components: + raise ValueError('Missing file path') + + file_path = os.path.join(*file_path_components) + pack_name = pack_db.name + + normalized_file_path = get_pack_file_abs_path(pack_name=pack_name, file_path=file_path) + + if not normalized_file_path or not os.path.isfile(normalized_file_path): + # Ignore references to files which don't exist on disk + raise StackStormDBObjectNotFoundError('File "%s" not found' % (file_path)) + + content = self._get_file_content(file_path=normalized_file_path) + return content + + +class PackViewsController(RestController): + files = FilesController() + file = FileController() diff --git a/st2api/tests/unit/controllers/v1/test_actions.py b/st2api/tests/unit/controllers/v1/test_actions.py index 96748a051b..a32717bd4f 100644 --- a/st2api/tests/unit/controllers/v1/test_actions.py +++ b/st2api/tests/unit/controllers/v1/test_actions.py @@ -13,7 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import os.path import copy + try: import simplejson as json except ImportError: @@ -25,6 +28,9 @@ from st2common.persistence.action import Action import st2common.validators.api.action as action_validator from st2common.constants.pack import SYSTEM_PACK_NAME +from st2common.persistence.pack import Pack +from st2tests.fixturesloader import get_fixtures_base_path +from st2tests.base import CleanFilesTestCase from tests import FunctionalTest @@ -184,8 +190,31 @@ } } +# Good action inside dummy pack +ACTION_12 = { + 'name': 'st2.dummy.action1', + 'description': 'test description', + 'enabled': True, + 'pack': 'dummy_pack_1', + 'entry_point': '/tmp/test/action1.sh', + 'runner_type': 'local-shell-script', + 'parameters': { + 'a': {'type': 'string', 'default': 'A1'}, + 'b': {'type': 'string', 'default': 'B1'} + }, + 'tags': [ + {'name': 'tag1', 'value': 'dont-care'}, + {'name': 'tag2', 'value': 'dont-care'} + ] +} + + +class TestActionController(FunctionalTest, CleanFilesTestCase): + register_packs = True + to_delete_files = [ + os.path.join(get_fixtures_base_path(), 'dummy_pack_1/actions/filea.txt') + ] -class TestActionController(FunctionalTest): @mock.patch.object(action_validator, 'validate_action', mock.MagicMock( return_value=True)) def test_get_one_using_id(self): @@ -331,6 +360,31 @@ def test_post_duplicate(self): for i in action_ids: self.__do_delete(i) + @mock.patch.object(action_validator, 'validate_action', mock.MagicMock( + return_value=True)) + def test_post_include_files(self): + # Verify initial state + pack_db = Pack.get_by_ref(ACTION_12['pack']) + self.assertTrue('actions/filea.txt' not in pack_db.files) + + action = copy.deepcopy(ACTION_12) + action['data_files'] = [ + { + 'file_path': 'filea.txt', + 'content': 'test content' + } + ] + post_resp = self.__do_post(action) + + # Verify file has been written on disk + for file_path in self.to_delete_files: + self.assertTrue(os.path.exists(file_path)) + + # Verify PackDB.files has been updated + pack_db = Pack.get_by_ref(ACTION_12['pack']) + self.assertTrue('actions/filea.txt' in pack_db.files) + self.__do_delete(self.__get_action_id(post_resp)) + @mock.patch.object(action_validator, 'validate_action', mock.MagicMock( return_value=True)) def test_post_put_delete(self): diff --git a/st2common/st2common/bootstrap/base.py b/st2common/st2common/bootstrap/base.py index dff3499884..442e05b084 100644 --- a/st2common/st2common/bootstrap/base.py +++ b/st2common/st2common/bootstrap/base.py @@ -24,6 +24,7 @@ from st2common.content.loader import ContentPackLoader from st2common.models.api.pack import PackAPI from st2common.persistence.pack import Pack +from st2common.util.file_system import get_file_list __all__ = [ 'ResourceRegistrar' @@ -37,11 +38,21 @@ # a long running process. REGISTERED_PACKS_CACHE = {} +EXCLUDE_FILE_PATTERNS = [ + '*.pyc' +] + class ResourceRegistrar(object): ALLOWED_EXTENSIONS = [] - def __init__(self): + def __init__(self, use_pack_cache=True): + """ + :param use_pack_cache: True to cache which packs have been registered in memory and making + sure packs are only registered once. + :type use_pack_cache: ``bool`` + """ + self._use_pack_cache = use_pack_cache self._meta_loader = MetaLoader() self._pack_loader = ContentPackLoader() @@ -78,7 +89,7 @@ def register_pack(self, pack_name, pack_dir): """ Register pack in the provided directory. """ - if pack_name in REGISTERED_PACKS_CACHE: + if self._use_pack_cache and pack_name in REGISTERED_PACKS_CACHE: # This pack has already been registered during this register content run return @@ -110,6 +121,11 @@ def _register_pack(self, pack_name, pack_dir): raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) content['ref'] = pack_name + + # Include a list of pack files + pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) + content['files'] = pack_file_list + pack_api = PackAPI(**content) pack_db = PackAPI.to_model(pack_api) diff --git a/st2common/st2common/constants/triggers.py b/st2common/st2common/constants/triggers.py index fe0423b518..9e6f56d4a9 100644 --- a/st2common/st2common/constants/triggers.py +++ b/st2common/st2common/constants/triggers.py @@ -26,6 +26,7 @@ 'ACTION_SENSOR_TRIGGER', 'NOTIFY_TRIGGER', + 'ACTION_FILE_WRITTEN_TRIGGER', 'TIMER_TRIGGER_TYPES', 'INTERNAL_TRIGGER_TYPES', @@ -49,6 +50,20 @@ } } } +ACTION_FILE_WRITTEN_TRIGGER = { + 'name': 'st2.action.file_writen', + 'pack': SYSTEM_PACK_NAME, + 'description': 'Trigger encapsulating action file being written on disk.', + 'payload_schema': { + 'type': 'object', + 'properties': { + 'ref': {}, + 'file_path': {}, + 'content': {}, + 'host_info': {} + } + } +} NOTIFY_TRIGGER = { 'name': 'st2.generic.notifytrigger', @@ -148,7 +163,8 @@ INTERNAL_TRIGGER_TYPES = { 'action': [ ACTION_SENSOR_TRIGGER, - NOTIFY_TRIGGER + NOTIFY_TRIGGER, + ACTION_FILE_WRITTEN_TRIGGER ], 'sensor': [ SENSOR_SPAWN_TRIGGER, diff --git a/st2common/st2common/content/utils.py b/st2common/st2common/content/utils.py index 4c20bd3fa5..89b4d2a3a3 100644 --- a/st2common/st2common/content/utils.py +++ b/st2common/st2common/content/utils.py @@ -14,6 +14,7 @@ # limitations under the License. import os +import os.path from oslo_config import cfg @@ -26,6 +27,9 @@ 'get_packs_base_paths', 'get_pack_base_path', 'get_pack_directory', + 'get_pack_file_abs_path', + 'get_pack_resource_file_abs_path', + 'get_relative_path_to_pack', 'check_pack_directory_exists', 'check_pack_content_directory_exists' ] @@ -176,15 +180,117 @@ def get_entry_point_abs_path(pack=None, entry_point=None): :rtype: ``str`` """ - if entry_point is not None and len(entry_point) > 0: - if os.path.isabs(entry_point): - return entry_point + if not entry_point: + return None + if os.path.isabs(entry_point): pack_base_path = get_pack_base_path(pack_name=pack) - entry_point_abs_path = os.path.join(pack_base_path, 'actions', quote_unix(entry_point)) - return entry_point_abs_path + common_prefix = os.path.commonprefix([pack_base_path, entry_point]) + + if common_prefix != pack_base_path: + raise ValueError('Entry point file "%s" is located outside of the pack directory' % + (entry_point)) + + return entry_point + + entry_point_abs_path = get_pack_resource_file_abs_path(pack_name=pack, + resource_type='action', + file_path=entry_point) + return entry_point_abs_path + + +def get_pack_file_abs_path(pack_name, file_path): + """ + Retrieve full absolute path to the pack file. + + Note: This function also takes care of sanitizing ``file_name`` argument + preventing directory traversal and similar attacks. + + :param pack_name: Pack name. + :type pack_name: ``str`` + + :pack file_path: Resource file path relative to the pack directory (e.g. my_file.py or + actions/directory/my_file.py) + :type file_path: ``str`` + + :rtype: ``str`` + """ + pack_base_path = get_pack_base_path(pack_name=pack_name) + + path_components = [] + path_components.append(pack_base_path) + + # Normalize the path to prevent directory traversal + normalized_file_path = os.path.normpath('/' + file_path).lstrip('/') + + if normalized_file_path != file_path: + raise ValueError('Invalid file path: %s' % (file_path)) + + path_components.append(normalized_file_path) + result = os.path.join(*path_components) + + assert normalized_file_path in result + + # Final safety check for common prefix to avoid traversal attack + common_prefix = os.path.commonprefix([pack_base_path, result]) + if common_prefix != pack_base_path: + raise ValueError('Invalid file_path: %s' % (file_path)) + + return result + + +def get_pack_resource_file_abs_path(pack_name, resource_type, file_path): + """ + Retrieve full absolute path to the pack resource file. + + Note: This function also takes care of sanitizing ``file_name`` argument + preventing directory traversal and similar attacks. + + :param pack_name: Pack name. + :type pack_name: ``str`` + + :param resource_type: Pack resource type (e.g. action, sensor, etc.). + :type resource_type: ``str`` + + :pack file_path: Resource file path relative to the pack directory (e.g. my_file.py or + directory/my_file.py) + :type file_path: ``str`` + + :rtype: ``str`` + """ + path_components = [] + if resource_type == 'action': + path_components.append('actions/') + elif resource_type == 'sensor': + path_components.append('sensors/') + elif resource_type == 'rule': + path_components.append('rules/') else: - return None + raise ValueError('Invalid resource type: %s' % (resource_type)) + + path_components.append(file_path) + file_path = os.path.join(*path_components) + result = get_pack_file_abs_path(pack_name=pack_name, file_path=file_path) + return result + + +def get_relative_path_to_pack(pack_name, file_path): + """ + Retrieve a file path which is relative to the provided pack directory. + + :rtype: ``str`` + """ + pack_base_path = get_pack_base_path(pack_name=pack_name) + + if not os.path.isabs(file_path): + return file_path + + common_prefix = os.path.commonprefix([pack_base_path, file_path]) + if common_prefix != pack_base_path: + raise ValueError('file_path is not located inside the pack directory') + + relative_path = os.path.relpath(file_path, common_prefix) + return relative_path def get_action_libs_abs_path(pack=None, entry_point=None): diff --git a/st2common/st2common/models/api/action.py b/st2common/st2common/models/api/action.py index d5bf837a37..fc92c06dc5 100644 --- a/st2common/st2common/models/api/action.py +++ b/st2common/st2common/models/api/action.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy + from st2common.util import isotime from st2common.util import schema as util_schema from st2common import log as logging @@ -28,9 +30,12 @@ from st2common.models.system.common import ResourceReference -__all__ = ['ActionAPI', - 'LiveActionAPI', - 'RunnerTypeAPI'] +__all__ = [ + 'ActionAPI', + 'ActionCreateAPI', + 'LiveActionAPI', + 'RunnerTypeAPI' +] LOG = logging.getLogger(__name__) @@ -119,7 +124,9 @@ def to_model(cls, runner_type): class ActionAPI(BaseAPI): - """The system entity that represents a Stack Action/Automation in the system.""" + """ + The system entity that represents a Stack Action/Automation in the system. + """ model = ActionDB schema = { @@ -235,6 +242,32 @@ def to_model(cls, action): return model +class ActionCreateAPI(ActionAPI): + """ + API model for create action operations. + """ + schema = copy.deepcopy(ActionAPI.schema) + schema['properties']['data_files'] = { + 'description': 'Optional action script and data files which are written to the filesystem.', + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'file_path': { + 'type': 'string', + 'required': True + }, + 'content': { + 'type': 'string', + 'required': True + }, + }, + 'additionalProperties': False + }, + 'default': {} + } + + class LiveActionAPI(BaseAPI): """The system entity that represents the execution of a Stack Action/Automation in the system. diff --git a/st2common/st2common/models/api/base.py b/st2common/st2common/models/api/base.py index b8de82ff5f..0ad317de2c 100644 --- a/st2common/st2common/models/api/base.py +++ b/st2common/st2common/models/api/base.py @@ -179,7 +179,10 @@ def callfunction(*args, **kwargs): # Invalid number of arguments passed to the function meaning invalid path was # requested # Note: The check is hacky, but it works for now. - if re.search('takes exactly \d+ arguments \(\d+ given\)', message): + func_name = f.__name__ + pattern = '%s\(\) takes exactly \d+ arguments \(\d+ given\)' % (func_name) + + if re.search(pattern, message): raise exc.HTTPNotFound() else: raise e diff --git a/st2common/st2common/models/api/pack.py b/st2common/st2common/models/api/pack.py index 9a14b9146b..1f295f13e3 100644 --- a/st2common/st2common/models/api/pack.py +++ b/st2common/st2common/models/api/pack.py @@ -54,6 +54,11 @@ class PackAPI(BaseAPI): }, 'email': { 'type': 'string' + }, + 'files': { + 'type': 'array', + 'items': {'type': 'string'}, + 'default': [] } }, 'additionalProperties': False @@ -68,8 +73,9 @@ def to_model(cls, pack): version = str(pack.version) author = pack.author email = pack.email + files = getattr(pack, 'files', []) model = cls.model(name=name, description=description, ref=ref, keywords=keywords, - version=version, author=author, email=email) + version=version, author=author, email=email, files=files) return model diff --git a/st2common/st2common/models/db/pack.py b/st2common/st2common/models/db/pack.py index c561ddd757..3e9f08406c 100644 --- a/st2common/st2common/models/db/pack.py +++ b/st2common/st2common/models/db/pack.py @@ -34,6 +34,7 @@ class PackDB(stormbase.StormFoundationDB): version = me.StringField(required=True) # TODO: Enforce format author = me.StringField(required=True) email = me.EmailField(required=True) + files = me.ListField(field=me.StringField()) # specialized access objects pack_access = MongoDBAccess(PackDB) diff --git a/st2common/st2common/util/file_system.py b/st2common/st2common/util/file_system.py new file mode 100644 index 0000000000..a68183bbee --- /dev/null +++ b/st2common/st2common/util/file_system.py @@ -0,0 +1,71 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +File system related utility functions. +""" + +import os +import os.path +import fnmatch + +__all__ = [ + 'get_file_list' +] + + +def get_file_list(directory, exclude_patterns=None): + """ + Recurisvely retrieve a list of files in the provided directory. + + :param directory: Path to directory to retrieve the file list for. + :type directory: ``str`` + + :param exclude_patterns: A list of `fnmatch` compatible patterns of files to exclude from + the result. + :type exclude_patterns: ``list`` + + :return: List of files in the provided directory. Each file path is relative + to the provided directory. + :rtype: ``list`` + """ + result = [] + if not directory.endswith('/'): + # Make sure trailing slash is present + directory = directory + '/' + + def include_file(file_path): + if not exclude_patterns: + return True + + for exclude_pattern in exclude_patterns: + if fnmatch.fnmatch(file_path, exclude_pattern): + return False + + return True + + for (dirpath, dirnames, filenames) in os.walk(directory): + base_path = dirpath.replace(directory, '') + + for filename in filenames: + if base_path: + file_path = os.path.join(base_path, filename) + else: + file_path = filename + + if include_file(file_path=file_path): + result.append(file_path) + + return result diff --git a/st2common/st2common/util/system_info.py b/st2common/st2common/util/system_info.py index 7a1284795f..9d20295f44 100644 --- a/st2common/st2common/util/system_info.py +++ b/st2common/st2common/util/system_info.py @@ -16,10 +16,22 @@ import os import socket +__all__ = [ + 'get_host_info', + 'get_process_info' +] + + +def get_host_info(): + host_info = { + 'hostname': socket.gethostname() + } + return host_info + def get_process_info(): - runner_info = { + process_info = { 'hostname': socket.gethostname(), 'pid': os.getpid() } - return runner_info + return process_info diff --git a/st2common/tests/unit/test_content_utils.py b/st2common/tests/unit/test_content_utils.py index 586c37679f..3ff7b609a0 100644 --- a/st2common/tests/unit/test_content_utils.py +++ b/st2common/tests/unit/test_content_utils.py @@ -13,11 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import os.path + import unittest2 from oslo_config import cfg -from st2common.content.utils import get_packs_base_paths, get_aliases_base_paths +from st2common.content.utils import get_packs_base_paths +from st2common.content.utils import get_aliases_base_paths +from st2common.content.utils import get_pack_resource_file_abs_path from st2tests import config as tests_config +from st2tests.fixturesloader import get_fixtures_base_path class ContentUtilsTestCase(unittest2.TestCase): @@ -71,3 +77,32 @@ def test_get_aliases_base_paths(self): cfg.CONF.content.aliases_base_paths = '/opt/path1:/opt/path2:/opt/path1:/opt/path2' result = get_aliases_base_paths() self.assertEqual(result, ['/opt/path1', '/opt/path2']) + + def test_get_pack_resource_file_abs_path(self): + # Mock the packs path to point to the fixtures directory + cfg.CONF.content.packs_base_paths = get_fixtures_base_path() + + # Invalid resource type + expected_msg = 'Invalid resource type: fooo' + self.assertRaisesRegexp(ValueError, expected_msg, get_pack_resource_file_abs_path, + pack_name='dummy_pack_1', + resource_type='fooo', + file_path='test.py') + + # Invalid paths (directory traversal and absolute paths) + file_paths = ['/tmp/foo.py', '../foo.py', '/etc/passwd', '../../foo.py'] + for file_path in file_paths: + expected_msg = 'Invalid file path: .*%s' % (file_path) + self.assertRaisesRegexp(ValueError, expected_msg, get_pack_resource_file_abs_path, + pack_name='dummy_pack_1', + resource_type='action', + file_path=file_path) + + # Valid paths + file_paths = ['foo.py', 'a/foo.py', 'a/b/foo.py'] + for file_path in file_paths: + expected = os.path.join(get_fixtures_base_path(), 'dummy_pack_1/actions', file_path) + result = get_pack_resource_file_abs_path(pack_name='dummy_pack_1', + resource_type='action', + file_path=file_path) + self.assertEqual(result, expected) diff --git a/st2common/tests/unit/test_util_file_system.py b/st2common/tests/unit/test_util_file_system.py new file mode 100644 index 0000000000..3aaf922f00 --- /dev/null +++ b/st2common/tests/unit/test_util_file_system.py @@ -0,0 +1,50 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import os.path + +import unittest2 + +from st2common.util.file_system import get_file_list + +CURRENT_DIR = os.path.dirname(__file__) +ST2TESTS_DIR = os.path.join(CURRENT_DIR, '../../../st2tests/st2tests') + + +class FileSystemUtilsTestCase(unittest2.TestCase): + def test_get_file_list(self): + # Standard exclude pattern + directory = os.path.join(ST2TESTS_DIR, 'policies') + expected = [ + 'mock_exception.py', + 'concurrency.py', + '__init__.py', + 'meta/mock_exception.yaml', + 'meta/concurrency.yaml', + 'meta/__init__.py' + ] + result = get_file_list(directory=directory, exclude_patterns=['*.pyc']) + self.assertItemsEqual(expected, result) + + # Custom exclude pattern + expected = [ + 'mock_exception.py', + 'concurrency.py', + '__init__.py', + 'meta/__init__.py' + ] + result = get_file_list(directory=directory, exclude_patterns=['*.pyc', '*.yaml']) + self.assertItemsEqual(expected, result) diff --git a/st2tests/st2tests/base.py b/st2tests/st2tests/base.py index 204e0a429c..4c227c4902 100644 --- a/st2tests/st2tests/base.py +++ b/st2tests/st2tests/base.py @@ -19,6 +19,7 @@ import json import os +import os.path import sys import shutil @@ -29,6 +30,8 @@ from st2common.exceptions.db import StackStormDBObjectConflictError from st2common.models.db import db_setup, db_teardown +from st2common.bootstrap.base import ResourceRegistrar +from st2common.content.utils import get_packs_base_paths import st2common.models.db.rule as rule_model import st2common.models.db.sensor as sensor_model import st2common.models.db.trigger as trigger_model @@ -131,6 +134,14 @@ def _drop_collections(cls): for model in ALL_MODELS: model.drop_collection() + @classmethod + def _register_packs(self): + """ + Register all the packs inside the fixtures directory. + """ + registrar = ResourceRegistrar(use_pack_cache=False) + registrar.register_packs(base_dirs=get_packs_base_paths()) + class DbTestCase(BaseDbTestCase): """ @@ -142,12 +153,16 @@ class DbTestCase(BaseDbTestCase): db_connection = None current_result = None + register_packs = False @classmethod def setUpClass(cls): BaseDbTestCase.setUpClass() cls._establish_connection_and_re_create_db() + if cls.register_packs: + cls._register_packs() + @classmethod def tearDownClass(cls): drop_db = True @@ -255,19 +270,33 @@ def setUp(self): class CleanFilesTestCase(TestCase): """ - Base test class which deletes specified files and directories on tearDown. + Base test class which deletes specified files and directories on setUp and `tearDown. """ to_delete_files = [] to_delete_directories = [] + def setUp(self): + super(CleanFilesTestCase, self).setUp() + self._delete_files() + def tearDown(self): + super(CleanFilesTestCase, self).tearDown() + self._delete_files() + + def _delete_files(self): for file_path in self.to_delete_files: + if not os.path.isfile(file_path): + continue + try: os.remove(file_path) except Exception: pass for file_path in self.to_delete_directories: + if not os.path.isdir(file_path): + continue + try: shutil.rmtree(file_path) except Exception: