Allow user to include arbitrary data files when creating a new action using the API

CHANGELOG.rst

@@ -30,6 +30,10 @@ in development
 * Fix bug in triggers emitted on key value pair changes and sensor spawn/exit. When
   dispatching those triggers, the reference used didn't contain the pack names
   which meant it was invalid and lookups in the rules engine would fail. (bug-fix)
+* Allow user to include files which are written on disk inside the action create API payload.
+  (new feature)
+* Allow user to retrieve content of a file inside a pack by using the new
+  ``/packs/views/files/`` API endpoint. (new feature)
 
 0.12.1 - July 31, 2015
 ----------------------

st2actions/tests/unit/test_runner_container_service.py

@@ -50,8 +50,9 @@ def test_get_entry_point_absolute_path(self):
         service = RunnerContainerService()
         orig_path = cfg.CONF.content.system_packs_base_path
         cfg.CONF.content.system_packs_base_path = '/tests/packs'
-        acutal_path = service.get_entry_point_abs_path(pack='foo', entry_point='/foo/bar.py')
-        self.assertEqual(acutal_path, '/foo/bar.py', 'Entry point path doesn\'t match.')
+        acutal_path = service.get_entry_point_abs_path(pack='foo',
+                                                       entry_point='/tests/packs/foo/bar.py')
+        self.assertEqual(acutal_path, '/tests/packs/foo/bar.py', 'Entry point path doesn\'t match.')
         cfg.CONF.content.system_packs_base_path = orig_path
 
     def test_get_entry_point_absolute_path_empty(self):
@@ -86,7 +87,8 @@ def test_get_action_libs_abs_path(self):
         self.assertEqual(acutal_path, expected_path, 'Action libs path doesn\'t match.')
 
         # entry point absolute.
-        acutal_path = service.get_action_libs_abs_path(pack='foo', entry_point='/tmp/foo.py')
-        expected_path = os.path.join('/tmp', ACTION_LIBS_DIR)
+        acutal_path = service.get_action_libs_abs_path(pack='foo',
+                                                       entry_point='/tests/packs/foo/tmp/foo.py')
+        expected_path = os.path.join('/tests/packs/foo/tmp', ACTION_LIBS_DIR)
         self.assertEqual(acutal_path, expected_path, 'Action libs path doesn\'t match.')
         cfg.CONF.content.system_packs_base_path = orig_path

st2api/st2api/controllers/resource.py

@@ -181,7 +181,7 @@ def _get_one_by_name_or_id(self, name_or_id, exclude_fields=None):
 
         from_model_kwargs = self._get_from_model_kwargs_for_request(request=pecan.request)
         result = self.model.from_model(instance, **from_model_kwargs)
-        LOG.debug('GET %s with name_or_odid=%s, client_result=%s', pecan.request.path, id, result)
+        LOG.debug('GET %s with name_or_id=%s, client_result=%s', pecan.request.path, id, result)
 
         return result
 
@@ -235,11 +235,11 @@ def get_one(self, ref_or_id):
     def get_all(self, **kwargs):
         return self._get_all(**kwargs)
 
-    def _get_one(self, ref_or_id):
+    def _get_one(self, ref_or_id, exclude_fields=None):
         LOG.info('GET %s with ref_or_id=%s', pecan.request.path, ref_or_id)
 
         try:
-            instance = self._get_by_ref_or_id(ref_or_id=ref_or_id)
+            instance = self._get_by_ref_or_id(ref_or_id=ref_or_id, exclude_fields=exclude_fields)
         except Exception as e:
             LOG.exception(e.message)
             pecan.abort(http_client.NOT_FOUND, e.message)
@@ -268,7 +268,7 @@ def _get_all(self, **kwargs):
 
         return result
 
-    def _get_by_ref_or_id(self, ref_or_id):
+    def _get_by_ref_or_id(self, ref_or_id, exclude_fields=None):
         """
         Retrieve resource object by an id of a reference.
 
@@ -283,31 +283,24 @@ def _get_by_ref_or_id(self, ref_or_id):
             is_reference = False
 
         if is_reference:
-            resource_db = self._get_by_ref(resource_ref=ref_or_id)
+            resource_db = self._get_by_ref(resource_ref=ref_or_id, exclude_fields=exclude_fields)
         else:
-            resource_db = self._get_by_id(resource_id=ref_or_id)
+            resource_db = self._get_by_id(resource_id=ref_or_id, exclude_fields=exclude_fields)
 
         if not resource_db:
             msg = 'Resource with a reference or id "%s" not found' % (ref_or_id)
             raise StackStormDBObjectNotFoundError(msg)
 
         return resource_db
 
-    def _get_by_id(self, resource_id):
-        try:
-            resource_db = self.access.get_by_id(resource_id)
-        except Exception:
-            resource_db = None
-
-        return resource_db
-
-    def _get_by_ref(self, resource_ref):
+    def _get_by_ref(self, resource_ref, exclude_fields=None):
         try:
             ref = ResourceReference.from_string_reference(ref=resource_ref)
         except Exception:
             return None
 
-        resource_db = self.access.query(name=ref.name, pack=ref.pack).first()
+        resource_db = self.access.query(name=ref.name, pack=ref.pack,
+                                        exclude_fields=exclude_fields).first()
         return resource_db
 
     def _get_filters(self, **kwargs):

st2api/st2api/controllers/v1/actions.py

@@ -13,10 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mongoengine import ValidationError
+import os
+import os.path
 
-from pecan import abort
 import six
+from pecan import abort
+from mongoengine import ValidationError
 
 # TODO: Encapsulate mongoengine errors in our persistence layer. Exceptions
 #       that bubble up to this layer should be core Python exceptions or
@@ -26,11 +28,19 @@
 from st2api.controllers.v1.actionviews import ActionViewsController
 from st2common import log as logging
 from st2common.constants.pack import DEFAULT_PACK_NAME
+from st2common.constants.triggers import ACTION_FILE_WRITTEN_TRIGGER
 from st2common.exceptions.apivalidation import ValueValidationException
 from st2common.models.api.base import jsexpose
 from st2common.persistence.action import Action
 from st2common.models.api.action import ActionAPI
+from st2common.models.api.action import ActionCreateAPI
+from st2common.persistence.pack import Pack
 from st2common.validators.api.misc import validate_not_part_of_system_pack
+from st2common.content.utils import get_pack_base_path
+from st2common.content.utils import get_pack_resource_file_abs_path
+from st2common.content.utils import get_relative_path_to_pack
+from st2common.transport.reactor import TriggerDispatcher
+from st2common.util.system_info import get_host_info
 import st2common.validators.api.action as action_validator
 
 http_client = six.moves.http_client
@@ -58,7 +68,11 @@ class ActionsController(resource.ContentPackResourceController):
 
     include_reference = True
 
-    @jsexpose(body_cls=ActionAPI, status_code=http_client.CREATED)
+    def __init__(self, *args, **kwargs):
+        super(ActionsController, self).__init__(*args, **kwargs)
+        self._trigger_dispatcher = TriggerDispatcher(LOG)
+
+    @jsexpose(body_cls=ActionCreateAPI, status_code=http_client.CREATED)
     def post(self, action):
         """
             Create a new action.
@@ -73,19 +87,32 @@ def post(self, action):
         validate_not_part_of_system_pack(action)
         action_validator.validate_action(action)
 
+        # Write pack data files to disk (if any are provided)
+        data_files = getattr(action, 'data_files', [])
+        written_data_files = []
+        if data_files:
+            written_data_files = self._handle_data_files(pack_name=action.pack,
+                                                         data_files=data_files)
+
         action_model = ActionAPI.to_model(action)
 
         LOG.debug('/actions/ POST verified ActionAPI object=%s', action)
         action_db = Action.add_or_update(action_model)
         LOG.debug('/actions/ POST saved ActionDB object=%s', action_db)
 
-        extra = {'action_db': action_db}
+        # Dispatch an internal trigger for each written data file. This way user
+        # automate comitting this files to git using StackStorm rule
+        if written_data_files:
+            self._dispatch_trigger_for_written_data_files(action_db=action_db,
+                                                          written_data_files=written_data_files)
+
+        extra = {'acion_db': action_db}
         LOG.audit('Action created. Action.id=%s' % (action_db.id), extra=extra)
         action_api = ActionAPI.from_model(action_db)
 
         return action_api
 
-    @jsexpose(arg_types=[str], body_cls=ActionAPI)
+    @jsexpose(arg_types=[str], body_cls=ActionCreateAPI)
     def put(self, action_ref_or_id, action):
         action_db = self._get_by_ref_or_id(ref_or_id=action_ref_or_id)
         action_id = action_db.id
@@ -97,6 +124,13 @@ def put(self, action_ref_or_id, action):
         validate_not_part_of_system_pack(action)
         action_validator.validate_action(action)
 
+        # Write pack data files to disk (if any are provided)
+        data_files = getattr(action, 'data_files', [])
+        written_data_files = []
+        if data_files:
+            written_data_files = self._handle_data_files(pack_name=action.pack,
+                                                         data_files=data_files)
+
         try:
             action_db = ActionAPI.to_model(action)
             action_db.id = action_id
@@ -106,6 +140,12 @@ def put(self, action_ref_or_id, action):
             abort(http_client.BAD_REQUEST, str(e))
             return
 
+        # Dispatch an internal trigger for each written data file. This way user
+        # automate comitting this files to git using StackStorm rule
+        if written_data_files:
+            self._dispatch_trigger_for_written_data_files(action_db=action_db,
+                                                          written_data_files=written_data_files)
+
         action_api = ActionAPI.from_model(action_db)
         LOG.debug('PUT /actions/ client_result=%s', action_api)
 
@@ -143,3 +183,90 @@ def delete(self, action_ref_or_id):
         extra = {'action_db': action_db}
         LOG.audit('Action deleted. Action.id=%s' % (action_db.id), extra=extra)
         return None
+
+    def _handle_data_files(self, pack_name, data_files):
+        """
+        Method for handling action data files.
+
+        This method performs two tasks:
+
+        1. Writes files to disk
+        2. Updates affected PackDB model
+        """
+        # Write files to disk
+        written_file_paths = self._write_data_files_to_disk(pack_name=pack_name,
+                                                            data_files=data_files)
+
+        # Update affected PackDB model (update a list of files)
+        # Update PackDB
+        self._update_pack_model(pack_name=pack_name, data_files=data_files,
+                                written_file_paths=written_file_paths)
+
+        return written_file_paths
+
+    def _write_data_files_to_disk(self, pack_name, data_files):
+        """
+        Write files to disk.
+        """
+        written_file_paths = []
+
+        for data_file in data_files:
+            file_path = data_file['file_path']
+            content = data_file['content']
+
+            file_path = get_pack_resource_file_abs_path(pack_name=pack_name,
+                                                        resource_type='action',
+                                                        file_path=file_path)
+
+            LOG.debug('Writing data file "%s" to "%s"' % (str(data_file), file_path))
+            self._write_data_file(pack_name=pack_name, file_path=file_path, content=content)
+            written_file_paths.append(file_path)
+
+        return written_file_paths
+
+    def _update_pack_model(self, pack_name, data_files, written_file_paths):
+        """
+        Update PackDB models (update files list).
+        """
+        file_paths = []  # A list of paths relative to the pack directory for new files
+        for file_path in written_file_paths:
+            file_path = get_relative_path_to_pack(pack_name=pack_name, file_path=file_path)
+            file_paths.append(file_path)
+
+        pack_db = Pack.get_by_ref(pack_name)
+        pack_db.files = set(pack_db.files)
+        pack_db.files.update(set(file_paths))
+        pack_db.files = list(pack_db.files)
+        pack_db = Pack.add_or_update(pack_db)
+
+        return pack_db
+
+    def _write_data_file(self, pack_name, file_path, content):
+        """
+        Write data file on disk.
+        """
+        # Throw if pack directory doesn't exist
+        pack_base_path = get_pack_base_path(pack_name=pack_name)
+        if not os.path.isdir(pack_base_path):
+            raise ValueError('Directory for pack "%s" doesn\'t exist' % (pack_name))
+
+        # Create pack sub-directory tree if it doesn't exist
+        directory = os.path.dirname(file_path)
+
+        if not os.path.isdir(directory):
+            os.makedirs(directory)
+
+        with open(file_path, 'w') as fp:
+            fp.write(content)
+
+    def _dispatch_trigger_for_written_data_files(self, action_db, written_data_files):
+        trigger = ACTION_FILE_WRITTEN_TRIGGER['name']
+        host_info = get_host_info()
+
+        for file_path in written_data_files:
+            payload = {
+                'ref': action_db.ref,
+                'file_path': file_path,
+                'host_info': host_info
+            }
+            self._trigger_dispatcher.dispatch(trigger=trigger, payload=payload)

st2api/st2api/controllers/v1/packs.py

@@ -15,6 +15,7 @@
 
 from st2common.models.api.base import jsexpose
 from st2api.controllers.resource import ResourceController
+from st2api.controllers.v1.packviews import PackViewsController
 from st2common.models.api.pack import PackAPI
 from st2common.persistence.pack import Pack
 
@@ -35,6 +36,9 @@ class PacksController(ResourceController):
         'sort': ['ref']
     }
 
+    # Nested controllers
+    views = PackViewsController()
+
     @jsexpose(arg_types=[str])
     def get_one(self, name_or_id):
         return self._get_one_by_name_or_id(name_or_id=name_or_id)