diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 72e1cf5b24..9041981db5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -106,6 +106,10 @@ Added Contributed by @Rand01ph +* Added support to override enabled parameter of resources. #5506 + + Contributed by Amanda McGuinness (@amanda11 Intive) + Fixed ~~~~~ diff --git a/st2actions/tests/unit/test_actions_registrar.py b/st2actions/tests/unit/test_actions_registrar.py index cc9da33299..d01b83a05f 100644 --- a/st2actions/tests/unit/test_actions_registrar.py +++ b/st2actions/tests/unit/test_actions_registrar.py @@ -112,7 +112,7 @@ def test_register_action_with_no_params(self): "generic", "actions", "action-with-no-parameters.yaml" ) - self.assertEqual(registrar._register_action("dummy", action_file), None) + self.assertEqual(registrar._register_action("dummy", action_file), False) @mock.patch.object( action_validator, "_is_valid_pack", mock.MagicMock(return_value=True) diff --git a/st2api/st2api/controllers/v1/packs.py b/st2api/st2api/controllers/v1/packs.py index 7aacae7471..0642c2f90e 100644 --- a/st2api/st2api/controllers/v1/packs.py +++ b/st2api/st2api/controllers/v1/packs.py @@ -201,10 +201,17 @@ def post(self, pack_register_request): pack_path = content_utils.get_pack_base_path(pack) try: - registered_count = registrar.register_from_pack( - pack_dir=pack_path - ) - result[name] += registered_count + res = registrar.register_from_pack(pack_dir=pack_path) + # Where overridding is supported return is tuple of + # (registered,overridden) else its just registered + # count return + if isinstance(res, tuple): + result[name] += res[0] + if res[1] != 0: + result[f"{name}(overridden)"] = res[1] + else: + result[name] += res + except ValueError as e: # Throw more user-friendly exception if requsted pack doesn't exist if re.match( @@ -219,10 +226,16 @@ def post(self, pack_register_request): raise e else: packs_base_paths = content_utils.get_packs_base_paths() - registered_count = registrar.register_from_packs( - base_dirs=packs_base_paths - ) - result[name] += registered_count + res = registrar.register_from_packs(base_dirs=packs_base_paths) + # Where overridding is supported return is tuple of + # (registered,overridden) else its just registered + # count return + if isinstance(res, tuple): + result[name] += res[0] + if res[1] != 0: + result[f"{name}(overridden)"] = res[1] + else: + result[name] += res return result diff --git a/st2common/st2common/bootstrap/actionsregistrar.py b/st2common/st2common/bootstrap/actionsregistrar.py index 4349812f05..a514bf0138 100644 --- a/st2common/st2common/bootstrap/actionsregistrar.py +++ b/st2common/st2common/bootstrap/actionsregistrar.py @@ -43,13 +43,14 @@ def register_from_packs(self, base_dirs): Discover all the packs in the provided directory and register actions from all of the discovered packs. - :return: Number of actions registered. - :rtype: ``int`` + :return: Number of actions registered, Number of actions overridden + :rtype: ``tuple`` """ # Register packs first self.register_packs(base_dirs=base_dirs) registered_count = 0 + overridden_count = 0 content = self._pack_loader.get_content( base_dirs=base_dirs, content_type="actions" ) @@ -63,8 +64,11 @@ def register_from_packs(self, base_dirs): "Registering actions from pack %s:, dir: %s", pack, actions_dir ) actions = self._get_actions_from_pack(actions_dir) - count = self._register_actions_from_pack(pack=pack, actions=actions) + count, overridden = self._register_actions_from_pack( + pack=pack, actions=actions + ) registered_count += count + overridden_count += overridden except Exception as e: if self._fail_on_failure: raise e @@ -73,14 +77,14 @@ def register_from_packs(self, base_dirs): "Failed registering all actions from pack: %s", actions_dir ) - return registered_count + return registered_count, overridden_count def register_from_pack(self, pack_dir): """ Register all the actions from the provided pack. - :return: Number of actions registered. - :rtype: ``int`` + :return: Number of actions registered, Number of actions overridden + :rtype: ``tuple`` """ pack_dir = pack_dir[:-1] if pack_dir.endswith("/") else pack_dir _, pack = os.path.split(pack_dir) @@ -92,6 +96,7 @@ def register_from_pack(self, pack_dir): self.register_pack(pack_name=pack, pack_dir=pack_dir) registered_count = 0 + overridden_count = 0 if not actions_dir: return registered_count @@ -99,7 +104,7 @@ def register_from_pack(self, pack_dir): try: actions = self._get_actions_from_pack(actions_dir=actions_dir) - registered_count = self._register_actions_from_pack( + registered_count, overridden_count = self._register_actions_from_pack( pack=pack, actions=actions ) except Exception as e: @@ -108,7 +113,7 @@ def register_from_pack(self, pack_dir): LOG.exception("Failed registering all actions from pack: %s", actions_dir) - return registered_count + return registered_count, overridden_count def _get_actions_from_pack(self, actions_dir): actions = self.get_resources_from_pack(resources_dir=actions_dir) @@ -142,6 +147,9 @@ def _register_action(self, pack, action): ) content["metadata_file"] = metadata_file + # Pass override information + altered = self._override_loader.override(pack, "actions", content) + action_api = ActionAPI(**content) try: @@ -214,12 +222,17 @@ def _register_action(self, pack, action): LOG.exception("Failed to write action to db %s.", model.name) raise + return altered + def _register_actions_from_pack(self, pack, actions): registered_count = 0 + overridden_count = 0 for action in actions: try: LOG.debug("Loading action from %s.", action) - self._register_action(pack=pack, action=action) + altered = self._register_action(pack=pack, action=action) + if altered: + overridden_count += 1 except Exception as e: if self._fail_on_failure: msg = 'Failed to register action "%s" from pack "%s": %s' % ( @@ -234,7 +247,7 @@ def _register_actions_from_pack(self, pack, actions): else: registered_count += 1 - return registered_count + return registered_count, overridden_count def register_actions( diff --git a/st2common/st2common/bootstrap/aliasesregistrar.py b/st2common/st2common/bootstrap/aliasesregistrar.py index d961b32e0c..b2c5d3b306 100644 --- a/st2common/st2common/bootstrap/aliasesregistrar.py +++ b/st2common/st2common/bootstrap/aliasesregistrar.py @@ -40,13 +40,14 @@ def register_from_packs(self, base_dirs): Discover all the packs in the provided directory and register aliases from all of the discovered packs. - :return: Number of aliases registered. - :rtype: ``int`` + :return: Tuple, Number of aliases registered, overridden. + :rtype: ``tuple`` """ # Register packs first self.register_packs(base_dirs=base_dirs) registered_count = 0 + overridden_count = 0 content = self._pack_loader.get_content( base_dirs=base_dirs, content_type="aliases" ) @@ -60,8 +61,11 @@ def register_from_packs(self, base_dirs): "Registering aliases from pack %s:, dir: %s", pack, aliases_dir ) aliases = self._get_aliases_from_pack(aliases_dir) - count = self._register_aliases_from_pack(pack=pack, aliases=aliases) + count, overridden = self._register_aliases_from_pack( + pack=pack, aliases=aliases + ) registered_count += count + overridden_count += overridden except Exception as e: if self._fail_on_failure: raise e @@ -70,14 +74,14 @@ def register_from_packs(self, base_dirs): "Failed registering all aliases from pack: %s", aliases_dir ) - return registered_count + return registered_count, overridden_count def register_from_pack(self, pack_dir): """ Register all the aliases from the provided pack. - :return: Number of aliases registered. - :rtype: ``int`` + :return: Tuple, Number of aliases registered, overridden + :rtype: ``tuple`` """ pack_dir = pack_dir[:-1] if pack_dir.endswith("/") else pack_dir _, pack = os.path.split(pack_dir) @@ -89,14 +93,15 @@ def register_from_pack(self, pack_dir): self.register_pack(pack_name=pack, pack_dir=pack_dir) registered_count = 0 + overridden_count = 0 if not aliases_dir: - return registered_count + return registered_count, overridden_count LOG.debug("Registering aliases from pack %s:, dir: %s", pack, aliases_dir) try: aliases = self._get_aliases_from_pack(aliases_dir=aliases_dir) - registered_count = self._register_aliases_from_pack( + registered_count, overridden_count = self._register_aliases_from_pack( pack=pack, aliases=aliases ) except Exception as e: @@ -104,9 +109,9 @@ def register_from_pack(self, pack_dir): raise e LOG.exception("Failed registering all aliases from pack: %s", aliases_dir) - return registered_count + return registered_count, overridden_count - return registered_count + return registered_count, overridden_count def _get_aliases_from_pack(self, aliases_dir): return self.get_resources_from_pack(resources_dir=aliases_dir) @@ -144,14 +149,17 @@ def _get_action_alias_db( else: content["metadata_file"] = metadata_file + # Pass override information + altered = self._override_loader.override(pack, "aliases", content) + action_alias_api = ActionAliasAPI(**content) action_alias_api.validate() action_alias_db = ActionAliasAPI.to_model(action_alias_api) - return action_alias_db + return action_alias_db, altered def _register_action_alias(self, pack, action_alias): - action_alias_db = self._get_action_alias_db( + action_alias_db, altered = self._get_action_alias_db( pack=pack, action_alias=action_alias ) @@ -181,14 +189,18 @@ def _register_action_alias(self, pack, action_alias): except Exception: LOG.exception("Failed to create action alias %s.", action_alias_db.name) raise + return altered def _register_aliases_from_pack(self, pack, aliases): registered_count = 0 + overridden_count = 0 for alias in aliases: try: LOG.debug("Loading alias from %s.", alias) - self._register_action_alias(pack, alias) + altered = self._register_action_alias(pack, alias) + if altered: + overridden_count += 1 except Exception as e: if self._fail_on_failure: msg = 'Failed to register alias "%s" from pack "%s": %s' % ( @@ -203,7 +215,7 @@ def _register_aliases_from_pack(self, pack, aliases): else: registered_count += 1 - return registered_count + return registered_count, overridden_count def register_aliases( diff --git a/st2common/st2common/bootstrap/base.py b/st2common/st2common/bootstrap/base.py index 1070a3af38..275b93f1dc 100644 --- a/st2common/st2common/bootstrap/base.py +++ b/st2common/st2common/bootstrap/base.py @@ -22,6 +22,7 @@ from st2common import log as logging from st2common.constants.pack import CONFIG_SCHEMA_FILE_NAME from st2common.content.loader import MetaLoader +from st2common.content.loader import OverrideLoader from st2common.content.loader import ContentPackLoader from st2common.models.api.pack import PackAPI from st2common.models.api.pack import ConfigSchemaAPI @@ -68,6 +69,7 @@ def __init__( self._fail_on_failure = fail_on_failure self._meta_loader = MetaLoader() + self._override_loader = OverrideLoader() self._pack_loader = ContentPackLoader() # Maps runner name -> RunnerTypeDB diff --git a/st2common/st2common/bootstrap/rulesregistrar.py b/st2common/st2common/bootstrap/rulesregistrar.py index a58c34e162..ac66008194 100644 --- a/st2common/st2common/bootstrap/rulesregistrar.py +++ b/st2common/st2common/bootstrap/rulesregistrar.py @@ -42,13 +42,14 @@ class RulesRegistrar(ResourceRegistrar): def register_from_packs(self, base_dirs): """ - :return: Number of rules registered. - :rtype: ``int`` + :return: Tuple, Number of rules registered, overridden + :rtype: ``tuple`` """ # Register packs first self.register_packs(base_dirs=base_dirs) registered_count = 0 + overridden_count = 0 content = self._pack_loader.get_content( base_dirs=base_dirs, content_type="rules" ) @@ -59,22 +60,23 @@ def register_from_packs(self, base_dirs): try: LOG.debug("Registering rules from pack: %s", pack) rules = self._get_rules_from_pack(rules_dir) - count = self._register_rules_from_pack(pack, rules) + count, override = self._register_rules_from_pack(pack, rules) registered_count += count + overridden_count += override except Exception as e: if self._fail_on_failure: raise e LOG.exception("Failed registering all rules from pack: %s", rules_dir) - return registered_count + return registered_count, overridden_count def register_from_pack(self, pack_dir): """ Register all the rules from the provided pack. - :return: Number of rules registered. - :rtype: ``int`` + :return: Number of rules registered, Number of rules overridden + :rtype: ``tuple`` """ pack_dir = pack_dir[:-1] if pack_dir.endswith("/") else pack_dir _, pack = os.path.split(pack_dir) @@ -86,27 +88,31 @@ def register_from_pack(self, pack_dir): self.register_pack(pack_name=pack, pack_dir=pack_dir) registered_count = 0 + overridden_count = 0 if not rules_dir: - return registered_count + return registered_count, overridden_count LOG.debug("Registering rules from pack %s:, dir: %s", pack, rules_dir) try: rules = self._get_rules_from_pack(rules_dir=rules_dir) - registered_count = self._register_rules_from_pack(pack=pack, rules=rules) + registered_count, overridden_count = self._register_rules_from_pack( + pack=pack, rules=rules + ) except Exception as e: if self._fail_on_failure: raise e LOG.exception("Failed registering all rules from pack: %s", rules_dir) - return registered_count + return registered_count, overridden_count def _get_rules_from_pack(self, rules_dir): return self.get_resources_from_pack(resources_dir=rules_dir) def _register_rules_from_pack(self, pack, rules): registered_count = 0 + overridden_count = 0 # TODO: Refactor this monstrosity for rule in rules: @@ -128,6 +134,9 @@ def _register_rules_from_pack(self, pack, rules): ) content["metadata_file"] = metadata_file + # Pass override information + altered = self._override_loader.override(pack, "rules", content) + rule_api = RuleAPI(**content) rule_api.validate() rule_db = RuleAPI.to_model(rule_api) @@ -198,8 +207,10 @@ def _register_rules_from_pack(self, pack, rules): LOG.exception("Failed registering rule from %s.", rule) else: registered_count += 1 + if altered: + overridden_count += 1 - return registered_count + return registered_count, overridden_count def register_rules( diff --git a/st2common/st2common/bootstrap/sensorsregistrar.py b/st2common/st2common/bootstrap/sensorsregistrar.py index 7476041da1..6105f1c13e 100644 --- a/st2common/st2common/bootstrap/sensorsregistrar.py +++ b/st2common/st2common/bootstrap/sensorsregistrar.py @@ -41,13 +41,14 @@ def register_from_packs(self, base_dirs): Discover all the packs in the provided directory and register sensors from all of the discovered packs. - :return: Number of sensors registered. - :rtype: ``int`` + :return: Number of sensors registered, overridde + :rtype: ``tuple`` """ # Register packs first self.register_packs(base_dirs=base_dirs) registered_count = 0 + overridden_count = 0 content = self._pack_loader.get_content( base_dirs=base_dirs, content_type="sensors" ) @@ -61,8 +62,11 @@ def register_from_packs(self, base_dirs): "Registering sensors from pack %s:, dir: %s", pack, sensors_dir ) sensors = self._get_sensors_from_pack(sensors_dir) - count = self._register_sensors_from_pack(pack=pack, sensors=sensors) + count, overridden = self._register_sensors_from_pack( + pack=pack, sensors=sensors + ) registered_count += count + overridden_count += overridden except Exception as e: if self._fail_on_failure: raise e @@ -73,7 +77,7 @@ def register_from_packs(self, base_dirs): six.text_type(e), ) - return registered_count + return registered_count, overridden_count def register_from_pack(self, pack_dir): """ @@ -92,14 +96,15 @@ def register_from_pack(self, pack_dir): self.register_pack(pack_name=pack, pack_dir=pack_dir) registered_count = 0 + overridden_count = 0 if not sensors_dir: - return registered_count + return registered_count, overridden_count LOG.debug("Registering sensors from pack %s:, dir: %s", pack, sensors_dir) try: sensors = self._get_sensors_from_pack(sensors_dir=sensors_dir) - registered_count = self._register_sensors_from_pack( + registered_count, overridden_count = self._register_sensors_from_pack( pack=pack, sensors=sensors ) except Exception as e: @@ -112,16 +117,19 @@ def register_from_pack(self, pack_dir): six.text_type(e), ) - return registered_count + return registered_count, overridden_count def _get_sensors_from_pack(self, sensors_dir): return self.get_resources_from_pack(resources_dir=sensors_dir) def _register_sensors_from_pack(self, pack, sensors): registered_count = 0 + overridden_count = 0 for sensor in sensors: try: - self._register_sensor_from_pack(pack=pack, sensor=sensor) + _, altered = self._register_sensor_from_pack(pack=pack, sensor=sensor) + if altered: + overridden_count = overridden_count + 1 except Exception as e: if self._fail_on_failure: msg = 'Failed to register sensor "%s" from pack "%s": %s' % ( @@ -138,7 +146,7 @@ def _register_sensors_from_pack(self, pack, sensors): LOG.debug('Sensor "%s" successfully registered', sensor) registered_count += 1 - return registered_count + return (registered_count, overridden_count) def _register_sensor_from_pack(self, pack, sensor): sensor_metadata_file_path = sensor @@ -167,6 +175,9 @@ def _register_sensor_from_pack(self, pack, sensor): ) content["metadata_file"] = metadata_file + # Pass override information + altered = self._override_loader.override(pack, "sensors", content) + sensors_dir = os.path.dirname(sensor_metadata_file_path) sensor_file_path = os.path.join(sensors_dir, entry_point) artifact_uri = "file://%s" % (sensor_file_path) @@ -191,7 +202,7 @@ def _register_sensor_from_pack(self, pack, sensor): except: LOG.exception("Failed creating sensor model for %s", sensor) - return sensor_model + return sensor_model, altered def register_sensors( diff --git a/st2common/st2common/constants/pack.py b/st2common/st2common/constants/pack.py index f782a6920c..c0a68abc07 100644 --- a/st2common/st2common/constants/pack.py +++ b/st2common/st2common/constants/pack.py @@ -16,6 +16,7 @@ __all__ = [ "PACKS_PACK_NAME", "PACK_REF_WHITELIST_REGEX", + "RESERVED_PACK_LIST", "PACK_RESERVED_CHARACTERS", "PACK_VERSION_SEPARATOR", "PACK_VERSION_REGEX", @@ -37,6 +38,9 @@ # A list of allowed characters for the pack name PACK_REF_WHITELIST_REGEX = r"^[a-z0-9_]+$" +# A list of reserved pack names that cannot be used +RESERVED_PACK_LIST = ["_global"] + # Check for a valid semver string PACK_VERSION_REGEX = r"^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-[\da-z\-]+(?:\.[\da-z\-]+)*)?(?:\+[\da-z\-]+(?:\.[\da-z\-]+)*)?$" # noqa diff --git a/st2common/st2common/content/bootstrap.py b/st2common/st2common/content/bootstrap.py index 9ae04664ad..888d33c8ef 100644 --- a/st2common/st2common/content/bootstrap.py +++ b/st2common/st2common/content/bootstrap.py @@ -193,13 +193,14 @@ def register_sensors(): fail_on_failure = not cfg.CONF.register.no_fail_on_failure registered_count = 0 + overridden_count = 0 try: LOG.info("=========================================================") LOG.info("############## Registering sensors ######################") LOG.info("=========================================================") with Timer(key="st2.register.sensors"): - registered_count = sensors_registrar.register_sensors( + (registered_count, overridden_count) = sensors_registrar.register_sensors( pack_dir=pack_dir, fail_on_failure=fail_on_failure ) except Exception as e: @@ -210,6 +211,7 @@ def register_sensors(): raise e LOG.info("Registered %s sensors." % (registered_count)) + LOG.info("%s sensors had their metadata overridden." % (overridden_count)) def register_runners(): @@ -245,13 +247,14 @@ def register_actions(): fail_on_failure = not cfg.CONF.register.no_fail_on_failure registered_count = 0 + overridden_count = 0 try: LOG.info("=========================================================") LOG.info("############## Registering actions ######################") LOG.info("=========================================================") with Timer(key="st2.register.actions"): - registered_count = actions_registrar.register_actions( + registered_count, overridden_count = actions_registrar.register_actions( pack_dir=pack_dir, fail_on_failure=fail_on_failure, use_runners_cache=True, @@ -264,6 +267,7 @@ def register_actions(): raise e LOG.info("Registered %s actions." % (registered_count)) + LOG.info("%s actions had their metadata overridden." % (overridden_count)) def register_rules(): @@ -272,6 +276,7 @@ def register_rules(): fail_on_failure = not cfg.CONF.register.no_fail_on_failure registered_count = 0 + overridden_count = 0 try: LOG.info("=========================================================") @@ -284,7 +289,7 @@ def register_rules(): try: with Timer(key="st2.register.rules"): - registered_count = rules_registrar.register_rules( + registered_count, overridden_count = rules_registrar.register_rules( pack_dir=pack_dir, fail_on_failure=fail_on_failure ) except Exception as e: @@ -295,6 +300,7 @@ def register_rules(): raise e LOG.info("Registered %s rules.", registered_count) + LOG.info("%s rules had their metadata overridden." % (overridden_count)) def register_aliases(): @@ -302,13 +308,14 @@ def register_aliases(): fail_on_failure = not cfg.CONF.register.no_fail_on_failure registered_count = 0 + overridden_count = 0 try: LOG.info("=========================================================") LOG.info("############## Registering aliases ######################") LOG.info("=========================================================") with Timer(key="st2.register.aliases"): - registered_count = aliases_registrar.register_aliases( + registered_count, overridden_count = aliases_registrar.register_aliases( pack_dir=pack_dir, fail_on_failure=fail_on_failure ) except Exception as e: @@ -318,6 +325,7 @@ def register_aliases(): LOG.warning("Failed to register aliases.", exc_info=True) LOG.info("Registered %s aliases.", registered_count) + LOG.info("%s aliases had their metadata overridden." % (overridden_count)) def register_policies(): diff --git a/st2common/st2common/content/loader.py b/st2common/st2common/content/loader.py index 3eed301bda..7e57ef6ec0 100644 --- a/st2common/st2common/content/loader.py +++ b/st2common/st2common/content/loader.py @@ -20,6 +20,7 @@ from yaml.parser import ParserError import six +from oslo_config import cfg from st2common import log as logging from st2common.constants.meta import ALLOWED_EXTS from st2common.constants.meta import PARSER_FUNCS @@ -28,7 +29,7 @@ if six.PY2: from io import open -__all__ = ["ContentPackLoader", "MetaLoader"] +__all__ = ["ContentPackLoader", "MetaLoader", "OverrideLoader"] LOG = logging.getLogger(__name__) @@ -268,3 +269,135 @@ def _load(self, parser_func, file_path): except ParserError: LOG.exception("Failed loading content from %s.", file_path) raise + + +class OverrideLoader(object): + """ + Class for loading pack override data + """ + + # Mapping of permitted override types to resource name + ALLOWED_OVERRIDE_TYPES = { + "sensors": "class_name", + "actions": "name", + "rules": "name", + "aliases": "name", + } + + ALLOWED_OVERRIDE_NAMES = [ + "enabled", + ] + + DEFAULT_OVERRIDE_VALUES = {"enabled": True} + + def override(self, pack_name, resource_type, content): + + """ + Loads override content for pack, and updates content + + :param pack_name: Name of pack + :type pack_name: ``str`` + :param resource_type: Type of resource loading + :type type: ``str`` + :param content: Content as loaded from meta information + :type content: ``object`` + :return: Whether data was overridden + :rtype: ``bool`` + """ + orig_content = content.copy() + if resource_type not in self.ALLOWED_OVERRIDE_TYPES.keys(): + raise ValueError( + f"Invalid override type of {resource_type} attempted for pack {pack_name}" + ) + + override_dir = os.path.join(cfg.CONF.system.base_path, "overrides") + # Apply global overrides + global_file = os.path.join(override_dir, "_global.yaml") + self._apply_override_file(global_file, pack_name, resource_type, content, True) + + # Apply pack overrides + override_file = os.path.join(override_dir, f"{pack_name}.yaml") + self._apply_override_file( + override_file, pack_name, resource_type, content, False + ) + if content == orig_content: + overridden = False + else: + # Need to account for defaults that might not have been set + for key in self.ALLOWED_OVERRIDE_NAMES: + if key not in orig_content.keys() and key in content.keys(): + orig_content[key] = self.DEFAULT_OVERRIDE_VALUES[key] + if content == orig_content: + overridden = False + else: + overridden = True + return overridden + + def _apply_override_file( + self, override_file, pack_name, resource_type, content, global_file + ): + + """ + Loads override content from override file + + :param override_file: Override filename + :type override_file: ``str`` + :param pack_name: Name of pack + :type pack_name: ``str`` + :param resource_type: Type of resource loading + :type type: ``str`` + :param content: Content as loaded from meta information + :type content: ``object`` + """ + + if not os.path.exists(override_file): + # No override file for pack + LOG.debug(f"No override file {override_file} found") + return + + # Read override file + file_name, file_ext = os.path.splitext(override_file) + overrides = self._load(PARSER_FUNCS[file_ext], override_file) + # Apply overrides + if resource_type in overrides: + type_override = overrides[resource_type] + name = content[self.ALLOWED_OVERRIDE_TYPES[resource_type]] + if "defaults" in type_override: + for key in type_override["defaults"]: + if key in self.ALLOWED_OVERRIDE_NAMES: + content[key] = type_override["defaults"][key] + LOG.debug( + f"Overridden {resource_type} {pack_name}.{name} {key} to default value of {content[key]} from {override_file}" + ) + else: + raise ValueError( + f"Override attempted with invalid default key {key} in pack {pack_name}" + ) + + if global_file: + # No exceptions required in global content file + return + + if "exceptions" in type_override: + if name in type_override["exceptions"]: + for key in type_override["exceptions"][name]: + if key in self.ALLOWED_OVERRIDE_NAMES: + content[key] = type_override["exceptions"][name][key] + LOG.debug( + f"Overridden {resource_type} {pack_name}.{name} {key} to exception value of {content[key]} from {override_file}" + ) + else: + raise ValueError( + f"Override attempted with invalid exceptions key {key} in pack {pack_name}" + ) + + def _load(self, parser_func, file_path): + with open(file_path, "r", encoding="utf-8") as fd: + try: + return parser_func(fd) + except ValueError: + LOG.exception("Failed loading content from %s.", file_path) + raise + except ParserError: + LOG.exception("Failed loading content from %s.", file_path) + raise diff --git a/st2common/st2common/util/pack.py b/st2common/st2common/util/pack.py index 4dadba87cb..0c13ccf347 100644 --- a/st2common/st2common/util/pack.py +++ b/st2common/st2common/util/pack.py @@ -24,6 +24,7 @@ from st2common.util import schema as util_schema from st2common.constants.pack import MANIFEST_FILE_NAME from st2common.constants.pack import PACK_REF_WHITELIST_REGEX +from st2common.constants.pack import RESERVED_PACK_LIST from st2common.content.loader import MetaLoader from st2common.persistence.pack import Pack from st2common.exceptions.apivalidation import ValueValidationException @@ -88,6 +89,10 @@ def get_pack_ref_from_metadata(metadata, pack_directory_name=None): ) raise ValueError(msg % (metadata["name"])) + if pack_ref in RESERVED_PACK_LIST: + raise ValueError( + f"{pack_ref} is a reserved name, and cannot be used as a pack name" + ) return pack_ref diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py index ad64f1e3af..20100369b0 100644 --- a/st2common/st2common/util/virtualenvs.py +++ b/st2common/st2common/util/virtualenvs.py @@ -29,6 +29,7 @@ from st2common import log as logging from st2common.constants.pack import PACK_REF_WHITELIST_REGEX from st2common.constants.pack import BASE_PACK_REQUIREMENTS +from st2common.constants.pack import RESERVED_PACK_LIST from st2common.util.shell import run_command from st2common.util.shell import quote_unix from st2common.util.compat import to_ascii @@ -74,6 +75,11 @@ def setup_pack_virtualenv( if not re.match(PACK_REF_WHITELIST_REGEX, pack_name): raise ValueError('Invalid pack name "%s"' % (pack_name)) + if pack_name in RESERVED_PACK_LIST: + raise ValueError( + f"Pack name {pack_name} is a reserved name, and cannot be used" + ) + base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, "virtualenvs/") virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name)) diff --git a/st2common/tests/resources/overrides/_global.yaml b/st2common/tests/resources/overrides/_global.yaml new file mode 100644 index 0000000000..c90008b58b --- /dev/null +++ b/st2common/tests/resources/overrides/_global.yaml @@ -0,0 +1,4 @@ +--- +sensors: + defaults: + enabled: false diff --git a/st2common/tests/resources/overrides/overpack1.yaml b/st2common/tests/resources/overrides/overpack1.yaml new file mode 100644 index 0000000000..236107cf0a --- /dev/null +++ b/st2common/tests/resources/overrides/overpack1.yaml @@ -0,0 +1,7 @@ +--- +actions: + defaults: + enabled: False + exceptions: + action2: + enabled: True diff --git a/st2common/tests/resources/overrides/overpack2.yaml b/st2common/tests/resources/overrides/overpack2.yaml new file mode 100644 index 0000000000..620f843cbe --- /dev/null +++ b/st2common/tests/resources/overrides/overpack2.yaml @@ -0,0 +1,11 @@ +--- +actions: + defaults: + enabled: False + rubbish: False + exceptions: + action2: + enabled: True +sensors: + defaults: + enabled: True diff --git a/st2common/tests/resources/overrides/overpack3.yaml b/st2common/tests/resources/overrides/overpack3.yaml new file mode 100644 index 0000000000..6bf3532cc8 --- /dev/null +++ b/st2common/tests/resources/overrides/overpack3.yaml @@ -0,0 +1,11 @@ +--- +actions: + defaults: + enabled: False + exceptions: + action2: + rubbish: True +sensors: + exceptions: + sensor1: + enabled: True diff --git a/st2common/tests/resources/overrides/overpack4.yaml b/st2common/tests/resources/overrides/overpack4.yaml new file mode 100644 index 0000000000..f4f132aa20 --- /dev/null +++ b/st2common/tests/resources/overrides/overpack4.yaml @@ -0,0 +1,4 @@ +--- +actions: + defaults: + enabled: False diff --git a/st2common/tests/unit/test_aliasesregistrar.py b/st2common/tests/unit/test_aliasesregistrar.py index 4f17246dcf..38d86a2f9f 100644 --- a/st2common/tests/unit/test_aliasesregistrar.py +++ b/st2common/tests/unit/test_aliasesregistrar.py @@ -33,9 +33,13 @@ class TestAliasRegistrar(DbTestCase): def test_alias_registration(self): - count = aliasesregistrar.register_aliases(pack_dir=ALIASES_FIXTURE_PACK_PATH) + count, overridden = aliasesregistrar.register_aliases( + pack_dir=ALIASES_FIXTURE_PACK_PATH + ) # expect all files to contain be aliases self.assertEqual(count, len(os.listdir(ALIASES_FIXTURE_PATH))) + # Nothing overridden + self.assertEqual(0, overridden) action_alias_dbs = ActionAlias.get_all() self.assertEqual(action_alias_dbs[0].metadata_file, "aliases/alias1.yaml") diff --git a/st2common/tests/unit/test_content_loader.py b/st2common/tests/unit/test_content_loader.py index 279d04dc4a..663d5f7844 100644 --- a/st2common/tests/unit/test_content_loader.py +++ b/st2common/tests/unit/test_content_loader.py @@ -15,6 +15,7 @@ from __future__ import absolute_import +from oslo_config import cfg import os import unittest2 @@ -31,8 +32,10 @@ from mock import Mock from st2common.content.loader import ContentPackLoader +from st2common.content.loader import OverrideLoader from st2common.content.loader import LOG from st2common.constants.meta import yaml_safe_load +from st2tests import config CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) RESOURCES_DIR = os.path.abspath(os.path.join(CURRENT_DIR, "../resources")) @@ -114,6 +117,104 @@ def test_get_content_from_pack_no_sensors(self): ) self.assertEqual(result, None) + def test_get_override_action_from_default(self): + config.parse_args() + cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + loader = OverrideLoader() + content = {"name": "action1", "enabled": True} + self.assertTrue(loader.override("overpack1", "actions", content)) + self.assertFalse(content["enabled"]) + content = {"name": "action1", "enabled": False} + self.assertFalse(loader.override("overpack1", "actions", content)) + self.assertFalse(content["enabled"]) + + def test_get_override_action_from_exception(self): + config.parse_args() + cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + loader = OverrideLoader() + content = {"name": "action2", "enabled": True} + self.assertFalse(loader.override("overpack1", "actions", content)) + self.assertTrue(content["enabled"]) + content = {"name": "action2", "enabled": False} + self.assertTrue(loader.override("overpack1", "actions", content)) + self.assertTrue(content["enabled"]) + + def test_get_override_action_from_default_no_exceptions(self): + config.parse_args() + cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + loader = OverrideLoader() + content = {"name": "action1", "enabled": True} + self.assertTrue(loader.override("overpack4", "actions", content)) + self.assertFalse(content["enabled"]) + content = {"name": "action2", "enabled": True} + self.assertTrue(loader.override("overpack4", "actions", content)) + self.assertFalse(content["enabled"]) + + def test_get_override_action_from_global_default_no_exceptions(self): + config.parse_args() + cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + loader = OverrideLoader() + content = {"class_name": "sensor1", "enabled": True} + self.assertTrue(loader.override("overpack1", "sensors", content)) + self.assertFalse(content["enabled"]) + + def test_get_override_action_from_global_overridden_by_pack(self): + config.parse_args() + cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + loader = OverrideLoader() + content = {"class_name": "sensor1", "enabled": True} + self.assertFalse(loader.override("overpack2", "sensors", content)) + self.assertTrue(content["enabled"]) + + def test_get_override_action_from_global_overridden_by_pack_exception(self): + config.parse_args() + cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + loader = OverrideLoader() + content = {"class_name": "sensor1", "enabled": True} + self.assertFalse(loader.override("overpack3", "sensors", content)) + self.assertTrue(content["enabled"]) + + def test_get_override_invalid_type(self): + config.parse_args() + cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + loader = OverrideLoader() + content = {"name": "action2", "enabled": True} + self.assertRaises( + ValueError, + loader.override, + pack_name="overpack1", + resource_type="wrongtype", + content=content, + ) + + def test_get_override_invalid_default_key(self): + config.parse_args() + cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + loader = OverrideLoader() + content = {"name": "action1", "enabled": True} + self.assertRaises( + ValueError, + loader.override, + pack_name="overpack2", + resource_type="actions", + content=content, + ) + + def test_get_override_invalid_exceptions_key(self): + config.parse_args() + cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + loader = OverrideLoader() + content = {"name": "action1", "enabled": True} + loader.override("overpack1", "actions", content) + content = {"name": "action2", "enabled": True} + self.assertRaises( + ValueError, + loader.override, + pack_name="overpack3", + resource_type="actions", + content=content, + ) + class YamlLoaderTestCase(unittest2.TestCase): def test_yaml_safe_load(self): diff --git a/st2common/tests/unit/test_util_pack.py b/st2common/tests/unit/test_util_pack.py index 0b476b7336..8e9dd59884 100644 --- a/st2common/tests/unit/test_util_pack.py +++ b/st2common/tests/unit/test_util_pack.py @@ -19,6 +19,7 @@ from st2common.models.db.pack import PackDB from st2common.util.pack import get_pack_common_libs_path_for_pack_db from st2common.util.pack import get_pack_warnings +from st2common.util.pack import get_pack_ref_from_metadata class PackUtilsTestCase(unittest2.TestCase): @@ -66,3 +67,21 @@ def test_get_pack_warnings_no_python(self): pack_metadata = {"name": "PackNone"} warning = get_pack_warnings(pack_metadata) self.assertEqual(None, warning) + + def test_get_pack_ref_from_meta_name_valid(self): + pack_metadata = {"name": "pack1"} + pack_ref = get_pack_ref_from_metadata(pack_metadata) + self.assertEqual("pack1", pack_ref) + + def test_get_pack_ref_from_meta_ref_valid(self): + pack_metadata = {"name": "Pack1", "ref": "pack1"} + pack_ref = get_pack_ref_from_metadata(pack_metadata) + self.assertEqual("pack1", pack_ref) + + def test_get_pack_ref_from_meta_ref_global(self): + pack_metadata = {"name": "Pack1", "ref": "_global"} + self.assertRaises(ValueError, get_pack_ref_from_metadata, pack_metadata) + + def test_get_pack_ref_from_meta_name_global(self): + pack_metadata = {"name": "_global"} + self.assertRaises(ValueError, get_pack_ref_from_metadata, pack_metadata) diff --git a/st2common/tests/unit/test_virtualenvs.py b/st2common/tests/unit/test_virtualenvs.py index 439801f67a..502f1df83c 100644 --- a/st2common/tests/unit/test_virtualenvs.py +++ b/st2common/tests/unit/test_virtualenvs.py @@ -378,3 +378,16 @@ def assertVirtualenvExists(self, virtualenv_dir): self.assertTrue(os.path.isdir(os.path.join(virtualenv_dir, "bin/"))) return True + + def test_setup_virtualenv_reserved_packname(self): + # Test a virtualenv update with pack which has global name + pack_name = "_global" + + self.assertRaises( + ValueError, + setup_pack_virtualenv, + pack_name=pack_name, + update=False, + include_setuptools=False, + include_wheel=False, + ) diff --git a/st2tests/st2tests/action_aliases.py b/st2tests/st2tests/action_aliases.py index 88f02f9642..3fd3dee80e 100644 --- a/st2tests/st2tests/action_aliases.py +++ b/st2tests/st2tests/action_aliases.py @@ -130,7 +130,7 @@ def _get_action_alias_db_by_name(self, name): ) aliases = registrar._get_aliases_from_pack(aliases_dir=aliases_path) for alias_path in aliases: - action_alias_db = registrar._get_action_alias_db( + action_alias_db, altered = registrar._get_action_alias_db( pack=pack, action_alias=alias_path, ignore_metadata_file_error=True )