diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7c271c570f..6770482b3a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -152,7 +152,7 @@ Added Contributed by @Kami. -* Added garbage collection for rule_enforcement and trace models #5596 +* Added garbage collection for rule_enforcement and trace models #5596/5602 Contributed by Amanda McGuinness (@amanda11 intive) diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index 518562f5c9..01f5387e71 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -167,9 +167,9 @@ dump_dir = /opt/stackstorm/exports/ logging = /etc/st2/logging.exporter.conf [garbagecollector] -# Action execution output objects (ones generated by action output streaming) older than this value (days) will be automatically deleted. +# Action execution output objects (ones generated by action output streaming) older than this value (days) will be automatically deleted. Defaults to 7. action_executions_output_ttl = 7 -# Action executions and related objects (live actions, action output objects) older than this value (days) will be automatically deleted. +# Action executions and related objects (live actions, action output objects) older than this value (days) will be automatically deleted. Defaults to None (disabled). action_executions_ttl = None # How often to check database for old data and perform garbage collection. collection_interval = 600 @@ -177,13 +177,13 @@ collection_interval = 600 logging = /etc/st2/logging.garbagecollector.conf # Set to True to perform garbage collection on Inquiries (based on the TTL value per Inquiry) purge_inquiries = False -# Rule enforcements older than this value (days) will be automatically deleted. -rule_enforcement_ttl = None +# Rule enforcements older than this value (days) will be automatically deleted. Defaults to None (disabled). +rule_enforcements_ttl = None # How long to wait / sleep (in seconds) between collection of different object types. sleep_delay = 2 -# Trace objects older than this value (days) will be automatically deleted. -trace_ttl = None -# Trigger instances older than this value (days) will be automatically deleted. +# Trace objects older than this value (days) will be automatically deleted. Defaults to None (disabled). +traces_ttl = None +# Trigger instances older than this value (days) will be automatically deleted. Defaults to None (disabled). trigger_instances_ttl = None [keyvalue] diff --git a/st2common/bin/st2-purge-rule-enforcement b/st2common/bin/st2-purge-rule-enforcements similarity index 94% rename from st2common/bin/st2-purge-rule-enforcement rename to st2common/bin/st2-purge-rule-enforcements index ac061dc626..ad66748034 100755 --- a/st2common/bin/st2-purge-rule-enforcement +++ b/st2common/bin/st2-purge-rule-enforcements @@ -16,7 +16,7 @@ import sys -from st2common.cmd.purge_rule_enforcement import main +from st2common.cmd.purge_rule_enforcements import main if __name__ == "__main__": sys.exit(main()) diff --git a/st2common/bin/st2-purge-trace b/st2common/bin/st2-purge-traces similarity index 95% rename from st2common/bin/st2-purge-trace rename to st2common/bin/st2-purge-traces index 28343ecc33..47a4944ea6 100755 --- a/st2common/bin/st2-purge-trace +++ b/st2common/bin/st2-purge-traces @@ -16,7 +16,7 @@ import sys -from st2common.cmd.purge_trace import main +from st2common.cmd.purge_traces import main if __name__ == "__main__": sys.exit(main()) diff --git a/st2common/setup.py b/st2common/setup.py index ef4ba6a727..66f6324264 100644 --- a/st2common/setup.py +++ b/st2common/setup.py @@ -53,8 +53,8 @@ "bin/st2-register-content", "bin/st2-purge-executions", "bin/st2-purge-trigger-instances", - "bin/st2-purge-trace", - "bin/st2-purge-rule-enforcement", + "bin/st2-purge-traces", + "bin/st2-purge-rule-enforcements", "bin/st2-run-pack-tests", "bin/st2ctl", "bin/st2-generate-symmetric-crypto-key", diff --git a/st2common/st2common/cmd/purge_rule_enforcement.py b/st2common/st2common/cmd/purge_rule_enforcements.py similarity index 96% rename from st2common/st2common/cmd/purge_rule_enforcement.py rename to st2common/st2common/cmd/purge_rule_enforcements.py index 22f37632be..adbe0c0ab7 100755 --- a/st2common/st2common/cmd/purge_rule_enforcement.py +++ b/st2common/st2common/cmd/purge_rule_enforcements.py @@ -35,7 +35,7 @@ from st2common.script_setup import teardown as common_teardown from st2common.constants.exit_codes import SUCCESS_EXIT_CODE from st2common.constants.exit_codes import FAILURE_EXIT_CODE -from st2common.garbage_collection.rule_enforcement import purge_rule_enforcement +from st2common.garbage_collection.rule_enforcement import purge_rule_enforcements __all__ = ["main"] @@ -71,7 +71,7 @@ def main(): # Purge models. try: - purge_rule_enforcement(logger=LOG, timestamp=timestamp) + purge_rule_enforcements(logger=LOG, timestamp=timestamp) except Exception as e: LOG.exception(six.text_type(e)) return FAILURE_EXIT_CODE diff --git a/st2common/st2common/cmd/purge_trace.py b/st2common/st2common/cmd/purge_traces.py similarity index 95% rename from st2common/st2common/cmd/purge_trace.py rename to st2common/st2common/cmd/purge_traces.py index db25ff8b76..37b848d6f1 100755 --- a/st2common/st2common/cmd/purge_trace.py +++ b/st2common/st2common/cmd/purge_traces.py @@ -35,7 +35,7 @@ from st2common.script_setup import teardown as common_teardown from st2common.constants.exit_codes import SUCCESS_EXIT_CODE from st2common.constants.exit_codes import FAILURE_EXIT_CODE -from st2common.garbage_collection.trace import purge_trace +from st2common.garbage_collection.trace import purge_traces __all__ = ["main"] @@ -71,7 +71,7 @@ def main(): # Purge models. try: - purge_trace(logger=LOG, timestamp=timestamp) + purge_traces(logger=LOG, timestamp=timestamp) except Exception as e: LOG.exception(six.text_type(e)) return FAILURE_EXIT_CODE diff --git a/st2common/st2common/garbage_collection/rule_enforcement.py b/st2common/st2common/garbage_collection/rule_enforcement.py index bf986a771d..0bb1848866 100644 --- a/st2common/st2common/garbage_collection/rule_enforcement.py +++ b/st2common/st2common/garbage_collection/rule_enforcement.py @@ -24,10 +24,10 @@ from st2common.persistence.rule_enforcement import RuleEnforcement from st2common.util import isotime -__all__ = ["purge_rule_enforcement"] +__all__ = ["purge_rule_enforcements"] -def purge_rule_enforcement(logger, timestamp): +def purge_rule_enforcements(logger, timestamp): """ :param timestamp: Rule enforcement instances older than this timestamp will be deleted. :type timestamp: ``datetime.datetime diff --git a/st2common/st2common/garbage_collection/trace.py b/st2common/st2common/garbage_collection/trace.py index a3109b3a56..41a16363f0 100644 --- a/st2common/st2common/garbage_collection/trace.py +++ b/st2common/st2common/garbage_collection/trace.py @@ -24,10 +24,10 @@ from st2common.persistence.trace import Trace from st2common.util import isotime -__all__ = ["purge_trace"] +__all__ = ["purge_traces"] -def purge_trace(logger, timestamp): +def purge_traces(logger, timestamp): """ :param timestamp: Trace instances older than this timestamp will be deleted. :type timestamp: ``datetime.datetime diff --git a/st2common/tests/unit/test_purge_rule_enforcement.py b/st2common/tests/unit/test_purge_rule_enforcement.py index cdc20f132d..f4ecb3ea14 100644 --- a/st2common/tests/unit/test_purge_rule_enforcement.py +++ b/st2common/tests/unit/test_purge_rule_enforcement.py @@ -17,7 +17,7 @@ import bson from st2common import log as logging -from st2common.garbage_collection.rule_enforcement import purge_rule_enforcement +from st2common.garbage_collection.rule_enforcement import purge_rule_enforcements from st2common.models.db.rule_enforcement import RuleEnforcementDB from st2common.persistence.rule_enforcement import RuleEnforcement from st2common.util import date as date_utils @@ -46,7 +46,7 @@ def test_no_timestamp_doesnt_delete(self): self.assertRaisesRegexp( ValueError, expected_msg, - purge_rule_enforcement, + purge_rule_enforcements, logger=LOG, timestamp=None, ) @@ -63,7 +63,7 @@ def test_purge(self): ) self.assertEqual(len(RuleEnforcement.get_all()), 2) - purge_rule_enforcement(logger=LOG, timestamp=now - timedelta(days=10)) + purge_rule_enforcements(logger=LOG, timestamp=now - timedelta(days=10)) self.assertEqual(len(RuleEnforcement.get_all()), 1) @staticmethod diff --git a/st2common/tests/unit/test_purge_trace.py b/st2common/tests/unit/test_purge_trace.py index 5ce472fa35..d734974f5e 100644 --- a/st2common/tests/unit/test_purge_trace.py +++ b/st2common/tests/unit/test_purge_trace.py @@ -17,7 +17,7 @@ import bson from st2common import log as logging -from st2common.garbage_collection.trace import purge_trace +from st2common.garbage_collection.trace import purge_traces from st2common.models.db.trace import TraceDB, TraceComponentDB from st2common.persistence.trace import Trace from st2common.util import date as date_utils @@ -50,7 +50,7 @@ def test_no_timestamp_doesnt_delete(self): self.assertRaisesRegexp( ValueError, expected_msg, - purge_trace, + purge_traces, logger=LOG, timestamp=None, ) @@ -75,7 +75,7 @@ def test_purge(self): ) self.assertEqual(len(Trace.get_all()), 2) - purge_trace(logger=LOG, timestamp=now - timedelta(days=10)) + purge_traces(logger=LOG, timestamp=now - timedelta(days=10)) self.assertEqual(len(Trace.get_all()), 1) @staticmethod diff --git a/st2reactor/st2reactor/garbage_collector/base.py b/st2reactor/st2reactor/garbage_collector/base.py index e0a651e7cb..9fa606825c 100644 --- a/st2reactor/st2reactor/garbage_collector/base.py +++ b/st2reactor/st2reactor/garbage_collector/base.py @@ -41,8 +41,8 @@ from st2common.garbage_collection.executions import purge_orphaned_workflow_executions from st2common.garbage_collection.inquiries import purge_inquiries from st2common.garbage_collection.trigger_instances import purge_trigger_instances -from st2common.garbage_collection.trace import purge_trace -from st2common.garbage_collection.rule_enforcement import purge_rule_enforcement +from st2common.garbage_collection.trace import purge_traces +from st2common.garbage_collection.rule_enforcement import purge_rule_enforcements __all__ = ["GarbageCollectorService"] @@ -71,8 +71,8 @@ def __init__( cfg.CONF.garbagecollector.action_executions_output_ttl ) self._trigger_instances_ttl = cfg.CONF.garbagecollector.trigger_instances_ttl - self._trace_ttl = cfg.CONF.garbagecollector.trace_ttl - self._rule_enforcement_ttl = cfg.CONF.garbagecollector.rule_enforcement_ttl + self._traces_ttl = cfg.CONF.garbagecollector.traces_ttl + self._rule_enforcements_ttl = cfg.CONF.garbagecollector.rule_enforcements_ttl self._purge_inquiries = cfg.CONF.garbagecollector.purge_inquiries self._workflow_execution_max_idle = cfg.CONF.workflow_engine.gc_max_idle_sec @@ -157,14 +157,17 @@ def _validate_ttl_values(self): ) % (MINIMUM_TTL_DAYS_EXECUTION_OUTPUT) ) - if self._trace_ttl and self._trace_ttl < MINIMUM_TTL_DAYS: + if self._traces_ttl and self._traces_ttl < MINIMUM_TTL_DAYS: raise ValueError( - "Minimum possible TTL for trace_ttl in days is %s" % (MINIMUM_TTL_DAYS) + "Minimum possible TTL for traces_ttl in days is %s" % (MINIMUM_TTL_DAYS) ) - if self._rule_enforcement_ttl and self._rule_enforcement_ttl < MINIMUM_TTL_DAYS: + if ( + self._rule_enforcements_ttl + and self._rule_enforcements_ttl < MINIMUM_TTL_DAYS + ): raise ValueError( - "Minimum possible TTL for rule_enforcement_ttl in days is %s" + "Minimum possible TTL for rule_enforcements_ttl in days is %s" % (MINIMUM_TTL_DAYS) ) @@ -214,9 +217,9 @@ def _perform_garbage_collection(self): obj_type = "trace" - if self._trace_ttl and self._trace_ttl >= MINIMUM_TTL_DAYS: + if self._traces_ttl and self._traces_ttl >= MINIMUM_TTL_DAYS: LOG.info(proc_message, obj_type) - self._purge_trace() + self._purge_traces() concurrency.sleep(self._sleep_delay) else: LOG.debug(skip_message, obj_type) @@ -224,11 +227,11 @@ def _perform_garbage_collection(self): obj_type = "rule enforcement" if ( - self._rule_enforcement_ttl - and self._rule_enforcement_ttl >= MINIMUM_TTL_DAYS + self._rule_enforcements_ttl + and self._rule_enforcements_ttl >= MINIMUM_TTL_DAYS ): LOG.info(proc_message, obj_type) - self._purge_rule_enforcement() + self._purge_rule_enforcements() concurrency.sleep(self._sleep_delay) else: LOG.debug(skip_message, obj_type) @@ -342,12 +345,12 @@ def _purge_trigger_instances(self): return True - def _purge_trace(self): + def _purge_traces(self): """ Purge trace objects which match the criteria defined in the config. """ utc_now = get_datetime_utc_now() - timestamp = utc_now - datetime.timedelta(days=self._trace_ttl) + timestamp = utc_now - datetime.timedelta(days=self._traces_ttl) # Another sanity check to make sure we don't delete new objects if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)): @@ -365,18 +368,18 @@ def _purge_trace(self): ) try: - purge_trace(logger=LOG, timestamp=timestamp) + purge_traces(logger=LOG, timestamp=timestamp) except Exception as e: LOG.exception("Failed to delete trace: %s" % (six.text_type(e))) return True - def _purge_rule_enforcement(self): + def _purge_rule_enforcements(self): """ Purge rule enforcements which match the criteria defined in the config. """ utc_now = get_datetime_utc_now() - timestamp = utc_now - datetime.timedelta(days=self._rule_enforcement_ttl) + timestamp = utc_now - datetime.timedelta(days=self._rule_enforcements_ttl) # Another sanity check to make sure we don't delete new objects if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)): @@ -394,7 +397,7 @@ def _purge_rule_enforcement(self): ) try: - purge_rule_enforcement(logger=LOG, timestamp=timestamp) + purge_rule_enforcements(logger=LOG, timestamp=timestamp) except Exception as e: LOG.exception("Failed to delete rule enforcements: %s" % (six.text_type(e))) diff --git a/st2reactor/st2reactor/garbage_collector/config.py b/st2reactor/st2reactor/garbage_collector/config.py index 01c1cee117..c70074939d 100644 --- a/st2reactor/st2reactor/garbage_collector/config.py +++ b/st2reactor/st2reactor/garbage_collector/config.py @@ -83,28 +83,28 @@ def _register_garbage_collector_opts(ignore_errors=False): "action_executions_ttl", default=None, help="Action executions and related objects (live actions, action output " - "objects) older than this value (days) will be automatically deleted.", + "objects) older than this value (days) will be automatically deleted. Defaults to None (disabled).", ), cfg.IntOpt( "action_executions_output_ttl", default=7, help="Action execution output objects (ones generated by action output " - "streaming) older than this value (days) will be automatically deleted.", + "streaming) older than this value (days) will be automatically deleted. Defaults to 7.", ), cfg.IntOpt( "trigger_instances_ttl", default=None, - help="Trigger instances older than this value (days) will be automatically deleted.", + help="Trigger instances older than this value (days) will be automatically deleted. Defaults to None (disabled).", ), cfg.IntOpt( - "rule_enforcement_ttl", + "rule_enforcements_ttl", default=None, - help="Rule enforcements older than this value (days) will be automatically deleted.", + help="Rule enforcements older than this value (days) will be automatically deleted. Defaults to None (disabled).", ), cfg.IntOpt( - "trace_ttl", + "traces_ttl", default=None, - help="Trace objects older than this value (days) will be automatically deleted.", + help="Trace objects older than this value (days) will be automatically deleted. Defaults to None (disabled).", ), ] diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py index 6ef73fd4ae..1d34159d8c 100644 --- a/st2tests/st2tests/config.py +++ b/st2tests/st2tests/config.py @@ -471,28 +471,28 @@ def _register_garbage_collector_opts(): "action_executions_ttl", default=None, help="Action executions and related objects (live actions, action output " - "objects) older than this value (days) will be automatically deleted.", + "objects) older than this value (days) will be automatically deleted. Defaults to None (disabled).", ), cfg.IntOpt( "action_executions_output_ttl", default=7, help="Action execution output objects (ones generated by action output " - "streaming) older than this value (days) will be automatically deleted.", + "streaming) older than this value (days) will be automatically deleted. Defaults to 7.", ), cfg.IntOpt( "trigger_instances_ttl", default=None, - help="Trigger instances older than this value (days) will be automatically deleted.", + help="Trigger instances older than this value (days) will be automatically deleted. Defaults to None (disabled).", ), cfg.IntOpt( - "rule_enforcement_ttl", + "rule_enforcements_ttl", default=None, - help="Rule enforcements older than this value (days) will be automatically deleted.", + help="Rule enforcements older than this value (days) will be automatically deleted. Defaults to None (disabled).", ), cfg.IntOpt( - "trace_ttl", + "traces_ttl", default=None, - help="Trace objects older than this value (days) will be automatically deleted.", + help="Trace objects older than this value (days) will be automatically deleted. Defaults to None (disabled).", ), ]