diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2d78b0efcb..ae16142a43 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -21,6 +21,8 @@ Docs: http://docks.stackstorm.com/latest actions were executed through SSH, now they are executed directly without the overhead of SSH. * Fix local runner so it correctly executes a command under the provider system user if ``user`` parameter is provided. (bug-fix) +* Fix a bug with a Trigger database object in some cases being created twice when registering a + rule. (bug-fix) v0.6.0 - December 8, 2014 ------------------------- diff --git a/contrib/examples/actions/mistral-basic.yaml b/contrib/examples/actions/mistral-basic.yaml index d2144cf2d3..d5f57dc44e 100644 --- a/contrib/examples/actions/mistral-basic.yaml +++ b/contrib/examples/actions/mistral-basic.yaml @@ -1,5 +1,6 @@ name: 'examples.mistral-basic' version: '2.0' +description: 'Basic mistral workflow example' workflows: demo: type: direct diff --git a/contrib/packs/actions/install.meta.yaml b/contrib/packs/actions/install.meta.yaml index ac5fc5be47..aab6bb19a5 100644 --- a/contrib/packs/actions/install.meta.yaml +++ b/contrib/packs/actions/install.meta.yaml @@ -5,7 +5,7 @@ Will download pack, load the actions, sensors and rules from the pack. Note that install require reboot of some st2 services." enabled: true - entry_point: "install.yaml" + entry_point: "workflows/install.yaml" parameters: packs: type: "array" diff --git a/contrib/packs/actions/uninstall.meta.yaml b/contrib/packs/actions/uninstall.meta.yaml index 98788e5cf4..469298dbf7 100644 --- a/contrib/packs/actions/uninstall.meta.yaml +++ b/contrib/packs/actions/uninstall.meta.yaml @@ -5,11 +5,10 @@ Removes pack and content from st2. Note that uninstall require reboot of some st2 services." enabled: true - entry_point: "uninstall.yaml" + entry_point: "workflows/uninstall.yaml" parameters: packs: type: "array" items: type: "string" required: true - diff --git a/contrib/packs/actions/install.yaml b/contrib/packs/actions/workflows/install.yaml similarity index 100% rename from contrib/packs/actions/install.yaml rename to contrib/packs/actions/workflows/install.yaml diff --git a/contrib/packs/actions/uninstall.yaml b/contrib/packs/actions/workflows/uninstall.yaml similarity index 100% rename from contrib/packs/actions/uninstall.yaml rename to contrib/packs/actions/workflows/uninstall.yaml diff --git a/st2common/st2common/models/system/action.py b/st2common/st2common/models/system/action.py index 4d3a72ef93..cb80f1d66f 100644 --- a/st2common/st2common/models/system/action.py +++ b/st2common/st2common/models/system/action.py @@ -51,6 +51,7 @@ def __init__(self, name, action_exec_id, command, user, env_vars=None, sudo=Fals self.env_vars = env_vars or {} self.user = user self.sudo = sudo + self.timeout = timeout def get_full_command_string(self): if self.sudo: @@ -250,6 +251,7 @@ def __init__(self, name, action_exec_id, script_local_path_abs, script_local_lib self.on_behalf_user = on_behalf_user self.remote_script = os.path.join(self.remote_dir, pipes.quote(self.script_name)) self.hosts = hosts + self.parallel = parallel self.command = self._format_command() LOG.debug('RemoteScriptAction: command to run on remote box: %s', self.command) diff --git a/st2common/st2common/services/triggers.py b/st2common/st2common/services/triggers.py index 7813b45e6d..de2e05e1d4 100644 --- a/st2common/st2common/services/triggers.py +++ b/st2common/st2common/services/triggers.py @@ -18,6 +18,17 @@ from st2common.models.system.common import ResourceReference from st2common.persistence.reactor import (Trigger, TriggerType) +__all__ = [ + 'get_trigger_db', + 'get_trigger_type_db', + + 'create_trigger_db', + 'create_trigger_type_db', + + 'create_or_update_trigger_db', + 'create_or_update_trigger_type_db' +] + LOG = logging.getLogger(__name__) @@ -92,12 +103,12 @@ def _get_trigger_api_given_rule(rule): trigger = rule.trigger triggertype_ref = ResourceReference.from_string_reference(trigger.get('type')) trigger_dict = {} - trigger_name = trigger.get('name', None) - if trigger_name: - trigger_dict['name'] = trigger_name + + trigger_dict['name'] = triggertype_ref.name trigger_dict['pack'] = triggertype_ref.pack trigger_dict['type'] = triggertype_ref.ref trigger_dict['parameters'] = rule.trigger.get('parameters', {}) + trigger_api = TriggerAPI(**trigger_dict) return trigger_api @@ -110,11 +121,44 @@ def create_trigger_db(trigger): trigger_db = get_trigger_db(trigger_api) if not trigger_db: trigger_db = TriggerAPI.to_model(trigger_api) - LOG.debug('verified trigger and formulated TriggerDB=%s', trigger_db) + LOG.debug('Verified trigger and formulated TriggerDB=%s', trigger_db) trigger_db = Trigger.add_or_update(trigger_db) return trigger_db +def create_or_update_trigger_db(trigger): + """ + Create a new TriggerDB model if one doesn't exist yet or update existing + one. + + :param trigger: Trigger info. + :type trigger: ``dict`` + """ + assert isinstance(trigger, dict) + + trigger_api = TriggerAPI(**trigger) + trigger_api = TriggerAPI.to_model(trigger_api) + + existing_trigger_db = get_trigger_db(trigger_api) + + if existing_trigger_db: + is_update = True + else: + is_update = False + + if is_update: + trigger_api.id = existing_trigger_db.id + + trigger_db = Trigger.add_or_update(trigger_api) + + if is_update: + LOG.audit('Trigger updated. Trigger=%s', trigger_db) + else: + LOG.audit('Trigger created. Trigger=%s', trigger_db) + + return trigger_db + + def create_trigger_db_from_rule(rule): trigger_api = _get_trigger_api_given_rule(rule) return create_trigger_db(trigger_api) @@ -133,8 +177,45 @@ def create_trigger_type_db(trigger_type): ref = ResourceReference.to_string_reference(name=trigger_type_api.name, pack=trigger_type_api.pack) trigger_type_db = get_trigger_type_db(ref) + if not trigger_type_db: trigger_type_db = TriggerTypeAPI.to_model(trigger_type_api) LOG.debug('verified trigger and formulated TriggerDB=%s', trigger_type_db) trigger_type_db = TriggerType.add_or_update(trigger_type_db) return trigger_type_db + + +def create_or_update_trigger_type_db(trigger_type): + """ + Create or update a trigger type db object in the db given trigger_type definition as dict. + + :param trigger_type: Trigger type model. + :type trigger_type: ``dict`` + + :rtype: ``object`` + """ + assert isinstance(trigger_type, dict) + + trigger_type_api = TriggerTypeAPI(**trigger_type) + trigger_type_api = TriggerTypeAPI.to_model(trigger_type_api) + + ref = ResourceReference.to_string_reference(name=trigger_type_api.name, + pack=trigger_type_api.pack) + + existing_trigger_type_db = get_trigger_type_db(ref) + if existing_trigger_type_db: + is_update = True + else: + is_update = False + + if is_update: + trigger_type_api.id = existing_trigger_type_db.id + + trigger_type_db = TriggerType.add_or_update(trigger_type_api) + + if is_update: + LOG.audit('TriggerType updated. TriggerType=%s', trigger_type_db) + else: + LOG.audit('TriggerType created. TriggerType=%s', trigger_type_db) + + return trigger_type_db diff --git a/st2common/tests/unit/test_fabric_system_model.py b/st2common/tests/unit/test_action_system_models.py similarity index 57% rename from st2common/tests/unit/test_fabric_system_model.py rename to st2common/tests/unit/test_action_system_models.py index 019112ba0e..7d988c7ecf 100644 --- a/st2common/tests/unit/test_fabric_system_model.py +++ b/st2common/tests/unit/test_action_system_models.py @@ -15,11 +15,57 @@ import unittest2 -from st2common.models.system.action import (FabricRemoteAction, FabricRemoteScriptAction) +from st2common.models.system.action import RemoteAction +from st2common.models.system.action import RemoteScriptAction +from st2common.models.system.action import FabricRemoteAction +from st2common.models.system.action import FabricRemoteScriptAction -class FabricRemoteActionsTest(unittest2.TestCase): +class RemoteActionTestCase(unittest2.TestCase): + def test_instantiation(self): + action = RemoteAction(name='name', action_exec_id='aeid', command='ls -la', + env_vars={'a': 1}, on_behalf_user='onbehalf', user='user', + hosts=['localhost'], parallel=False, sudo=True, timeout=10) + self.assertEqual(action.name, 'name') + self.assertEqual(action.action_exec_id, 'aeid') + self.assertEqual(action.command, 'ls -la') + self.assertEqual(action.env_vars, {'a': 1}) + self.assertEqual(action.on_behalf_user, 'onbehalf') + self.assertEqual(action.user, 'user') + self.assertEqual(action.hosts, ['localhost']) + self.assertEqual(action.parallel, False) + self.assertEqual(action.sudo, True) + self.assertEqual(action.timeout, 10) + +class RemoteScriptActionTestCase(unittest2.TestCase): + def test_instantiation(self): + action = RemoteScriptAction(name='name', action_exec_id='aeid', + script_local_path_abs='/tmp/sc/ma_script.sh', + script_local_libs_path_abs='/tmp/sc/libs', named_args=None, + positional_args=None, env_vars={'a': 1}, + on_behalf_user='onbehalf', user='user', + remote_dir='/home/mauser', hosts=['localhost'], + parallel=False, sudo=True, timeout=10) + self.assertEqual(action.name, 'name') + self.assertEqual(action.action_exec_id, 'aeid') + self.assertEqual(action.script_local_libs_path_abs, '/tmp/sc/libs') + self.assertEqual(action.env_vars, {'a': 1}) + self.assertEqual(action.on_behalf_user, 'onbehalf') + self.assertEqual(action.user, 'user') + self.assertEqual(action.remote_dir, '/home/mauser') + self.assertEqual(action.hosts, ['localhost']) + self.assertEqual(action.parallel, False) + self.assertEqual(action.sudo, True) + self.assertEqual(action.timeout, 10) + + self.assertEqual(action.script_local_dir, '/tmp/sc') + self.assertEqual(action.script_name, 'ma_script.sh') + self.assertEqual(action.remote_script, '/home/mauser/ma_script.sh') + self.assertEqual(action.command, '/home/mauser/ma_script.sh') + + +class FabricRemoteActionTestCase(unittest2.TestCase): def test_fabric_remote_action_method(self): remote_action = FabricRemoteAction('foo', 'foo-id', 'ls -lrth', on_behalf_user='stan', parallel=True, sudo=False) diff --git a/st2reactor/st2reactor/bootstrap/sensorsregistrar.py b/st2reactor/st2reactor/bootstrap/sensorsregistrar.py index dc1ebbd1e2..1c918ebb08 100644 --- a/st2reactor/st2reactor/bootstrap/sensorsregistrar.py +++ b/st2reactor/st2reactor/bootstrap/sensorsregistrar.py @@ -63,6 +63,10 @@ def _register_sensor_from_pack(self, pack, sensor): trigger_types = metadata.get('trigger_types', []) poll_interval = metadata.get('poll_interval', None) + # Add pack to each trigger type item + for trigger_type in trigger_types: + trigger_type['pack'] = pack + # Add TrigerType models to the DB trigger_type_dbs = container_utils.add_trigger_models(trigger_types=trigger_types) diff --git a/st2reactor/st2reactor/container/utils.py b/st2reactor/st2reactor/container/utils.py index 66032c903c..1ad6f38a05 100644 --- a/st2reactor/st2reactor/container/utils.py +++ b/st2reactor/st2reactor/container/utils.py @@ -49,8 +49,18 @@ def create_trigger_instance(trigger, payload, occurrence_time): return TriggerInstance.add_or_update(trigger_instance) -def _create_trigger_type(trigger_type): - return TriggerService.create_trigger_type_db(trigger_type) +def _create_trigger_type(pack, name, description=None, payload_schema=None, + parameters_schema=None): + trigger_type = { + 'name': name, + 'pack': pack, + 'description': description, + 'payload_schema': payload_schema, + 'parameters_schema': parameters_schema + } + + trigger_type_db = TriggerService.create_or_update_trigger_type_db(trigger_type=trigger_type) + return trigger_type_db def _validate_trigger_type(trigger_type): @@ -65,13 +75,25 @@ def _validate_trigger_type(trigger_type): def _create_trigger(trigger_type): + """ + :param trigger_type: TriggerType db object. + :type trigger_type: :class:`TriggerTypeDB` + """ if hasattr(trigger_type, 'parameters_schema') and not trigger_type['parameters_schema']: trigger_dict = { 'name': trigger_type.name, 'pack': trigger_type.pack, 'type': trigger_type.get_reference().ref } - return TriggerService.create_trigger_db(trigger_dict) + + try: + trigger_db = TriggerService.create_or_update_trigger_db(trigger=trigger_dict) + except: + LOG.exception('Validation failed for Trigger=%s.', trigger_dict) + raise TriggerTypeRegistrationException( + 'Unable to create Trigger for TriggerType=%s.' % trigger_type.name) + else: + return trigger_db else: LOG.debug('Won\'t create Trigger object as TriggerType %s expects ' + 'parameters.', trigger_type) @@ -79,7 +101,19 @@ def _create_trigger(trigger_type): def _add_trigger_models(trigger_type): - trigger_type = _create_trigger_type(trigger_type) + pack = trigger_type['pack'] + description = trigger_type['description'] if 'description' in trigger_type else '' + payload_schema = trigger_type['payload_schema'] if 'payload_schema' in trigger_type else {} + parameters_schema = trigger_type['parameters_schema'] \ + if 'parameters_schema' in trigger_type else {} + + trigger_type = _create_trigger_type( + pack=pack, + name=trigger_type['name'], + description=description, + payload_schema=payload_schema, + parameters_schema=parameters_schema + ) trigger = _create_trigger(trigger_type=trigger_type) return (trigger_type, trigger) @@ -89,7 +123,9 @@ def add_trigger_models(trigger_types): Register trigger types. :param trigger_types: A list of triggers to register. - :type trigger_types: ``list`` of trigger_type. + :type trigger_types: ``list`` of ``dict`` + + :rtype: ``list`` of ``tuple`` (trigger_type, trigger) """ [r for r in (_validate_trigger_type(trigger_type) for trigger_type in trigger_types) if r is not None] diff --git a/st2reactor/tests/fixtures/packs/pack_with_rules/rules/rule_with_timer.yaml b/st2reactor/tests/fixtures/packs/pack_with_rules/rules/rule_with_timer.yaml new file mode 100644 index 0000000000..f5cf652d9c --- /dev/null +++ b/st2reactor/tests/fixtures/packs/pack_with_rules/rules/rule_with_timer.yaml @@ -0,0 +1,15 @@ + +--- + name: "sample.with_timer" + description: "Sample rule using an Interval Timer." + trigger: + type: "core.st2.IntervalTimer" + parameters: + delta: 5 + unit: "seconds" + criteria: {} + action: + ref: "core.local" + parameters: + cmd: "echo \"{{trigger.executed_at}}\"" + enabled: true diff --git a/st2reactor/tests/fixtures/packs/pack_with_sensor/sensors/test_sensor.yaml b/st2reactor/tests/fixtures/packs/pack_with_sensor/sensors/test_sensor.yaml new file mode 100644 index 0000000000..11a6fd6992 --- /dev/null +++ b/st2reactor/tests/fixtures/packs/pack_with_sensor/sensors/test_sensor.yaml @@ -0,0 +1,12 @@ +--- + class_name: "TestSensor" + entry_point: "test_sensor.py" + description: "Test sensor" + poll_interval: 10 + trigger_types: + - + name: "trigger_type_1" + description: "1" + - + name: "trigger_type_2" + description: "2" diff --git a/st2reactor/tests/test_sensor_and_rule_registration.py b/st2reactor/tests/test_sensor_and_rule_registration.py new file mode 100644 index 0000000000..16f5e63b9e --- /dev/null +++ b/st2reactor/tests/test_sensor_and_rule_registration.py @@ -0,0 +1,136 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from st2tests import DbTestCase +from st2common.persistence.reactor import SensorType +from st2common.persistence.reactor import Trigger +from st2common.persistence.reactor import TriggerType +from st2common.persistence.reactor import Rule +from st2reactor.bootstrap.sensorsregistrar import SensorsRegistrar +from st2reactor.bootstrap.rulesregistrar import RulesRegistrar + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class SensorRegistrationTestCase(DbTestCase): + def setUp(self): + self._packs_base_path = os.path.join(CURRENT_DIR, 'fixtures/packs') + + def test_register_sensors(self): + # Verify DB is empty at the beginning + self.assertEqual(len(SensorType.get_all()), 0) + self.assertEqual(len(TriggerType.get_all()), 0) + self.assertEqual(len(Trigger.get_all()), 0) + + registrar = SensorsRegistrar() + registrar.register_sensors_from_packs(base_dir=self._packs_base_path) + + # Verify objects have been created + sensor_dbs = SensorType.get_all() + trigger_type_dbs = TriggerType.get_all() + trigger_dbs = Trigger.get_all() + + self.assertEqual(len(sensor_dbs), 1) + self.assertEqual(len(trigger_type_dbs), 2) + self.assertEqual(len(trigger_dbs), 2) + + self.assertEqual(sensor_dbs[0].name, 'TestSensor') + self.assertEqual(sensor_dbs[0].poll_interval, 10) + + self.assertEqual(trigger_type_dbs[0].name, 'trigger_type_1') + self.assertEqual(trigger_type_dbs[0].pack, 'pack_with_sensor') + self.assertEqual(trigger_type_dbs[1].name, 'trigger_type_2') + self.assertEqual(trigger_type_dbs[1].pack, 'pack_with_sensor') + + # Verify second call to registration doesn't create a duplicate objects + registrar.register_sensors_from_packs(base_dir=self._packs_base_path) + + sensor_dbs = SensorType.get_all() + trigger_type_dbs = TriggerType.get_all() + trigger_dbs = Trigger.get_all() + + self.assertEqual(len(sensor_dbs), 1) + self.assertEqual(len(trigger_type_dbs), 2) + self.assertEqual(len(trigger_dbs), 2) + + self.assertEqual(sensor_dbs[0].name, 'TestSensor') + self.assertEqual(sensor_dbs[0].poll_interval, 10) + + self.assertEqual(trigger_type_dbs[0].name, 'trigger_type_1') + self.assertEqual(trigger_type_dbs[0].pack, 'pack_with_sensor') + self.assertEqual(trigger_type_dbs[1].name, 'trigger_type_2') + self.assertEqual(trigger_type_dbs[1].pack, 'pack_with_sensor') + + # Verify sensor and trigger data is updated on registration + original_load = registrar._meta_loader.load + + def mock_load(*args, **kwargs): + # Update poll_interval and trigger_type_2 description + data = original_load(*args, **kwargs) + data['poll_interval'] = 50 + data['trigger_types'][1]['description'] = 'test 2' + return data + registrar._meta_loader.load = mock_load + + registrar.register_sensors_from_packs(base_dir=self._packs_base_path) + + sensor_dbs = SensorType.get_all() + trigger_type_dbs = TriggerType.get_all() + trigger_dbs = Trigger.get_all() + + self.assertEqual(len(sensor_dbs), 1) + self.assertEqual(len(trigger_type_dbs), 2) + self.assertEqual(len(trigger_dbs), 2) + + self.assertEqual(sensor_dbs[0].name, 'TestSensor') + self.assertEqual(sensor_dbs[0].poll_interval, 50) + + self.assertEqual(trigger_type_dbs[0].name, 'trigger_type_1') + self.assertEqual(trigger_type_dbs[0].pack, 'pack_with_sensor') + self.assertEqual(trigger_type_dbs[1].name, 'trigger_type_2') + self.assertEqual(trigger_type_dbs[1].pack, 'pack_with_sensor') + self.assertEqual(trigger_type_dbs[1].description, 'test 2') + + +class RuleRegistrationTestCase(DbTestCase): + def setUp(self): + self._packs_base_path = os.path.join(CURRENT_DIR, 'fixtures/packs') + + def test_register_rules(self): + # Verify DB is empty at the beginning + self.assertEqual(len(Rule.get_all()), 0) + self.assertEqual(len(Trigger.get_all()), 0) + + registrar = RulesRegistrar() + registrar.register_rules_from_packs(base_dir=self._packs_base_path) + + # Verify modeles are created + rule_dbs = Rule.get_all() + trigger_dbs = Trigger.get_all() + self.assertEqual(len(rule_dbs), 1) + self.assertEqual(len(trigger_dbs), 1) + + self.assertEqual(rule_dbs[0].name, 'sample.with_timer') + self.assertEqual(trigger_dbs[0].name, 'st2.IntervalTimer') + + # Verify second register call updates existing models + registrar.register_rules_from_packs(base_dir=self._packs_base_path) + + rule_dbs = Rule.get_all() + trigger_dbs = Trigger.get_all() + self.assertEqual(len(rule_dbs), 1) + self.assertEqual(len(trigger_dbs), 1)