From 5afe37d035cea6bbf5c6026ddabe9d9d97a87c1d Mon Sep 17 00:00:00 2001 From: Amanda McGuinness Date: Mon, 28 Mar 2022 11:03:16 +0000 Subject: [PATCH 1/2] Add garbage collection for trace and rule_enforcement models --- CHANGELOG.rst | 4 + conf/st2.conf.sample | 4 + st2common/bin/st2-purge-rule-enforcement | 22 ++++ st2common/bin/st2-purge-trace | 22 ++++ st2common/setup.py | 2 + .../st2common/cmd/purge_rule_enforcement.py | 81 ++++++++++++ st2common/st2common/cmd/purge_trace.py | 81 ++++++++++++ .../garbage_collection/rule_enforcement.py | 67 ++++++++++ .../st2common/garbage_collection/trace.py | 65 ++++++++++ .../st2common/persistence/rule_enforcement.py | 4 + st2common/st2common/persistence/trace.py | 4 + .../tests/unit/test_purge_rule_enforcement.py | 77 ++++++++++++ st2common/tests/unit/test_purge_trace.py | 117 ++++++++++++++++++ .../st2reactor/garbage_collector/base.py | 93 ++++++++++++++ .../st2reactor/garbage_collector/config.py | 10 ++ 15 files changed, 653 insertions(+) create mode 100755 st2common/bin/st2-purge-rule-enforcement create mode 100755 st2common/bin/st2-purge-trace create mode 100755 st2common/st2common/cmd/purge_rule_enforcement.py create mode 100755 st2common/st2common/cmd/purge_trace.py create mode 100644 st2common/st2common/garbage_collection/rule_enforcement.py create mode 100644 st2common/st2common/garbage_collection/trace.py create mode 100644 st2common/tests/unit/test_purge_rule_enforcement.py create mode 100644 st2common/tests/unit/test_purge_trace.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aa4439df03..ce435f7f93 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -152,6 +152,10 @@ Added Contributed by @Kami. +* Added garbage collection for rule_enforcement and trace models + Contributed by Amanda McGuinness (@amanda11 intive) + + Fixed ~~~~~ diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index 5dd450c2ad..60c8648bc2 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -181,6 +181,10 @@ purge_inquiries = False sleep_delay = 2 # Trigger instances older than this value (days) will be automatically deleted. trigger_instances_ttl = None +# Rule enforcement instances older than this value (days) will be automatically deleted. +rule_enforcement_ttl = None +# Trace instances older than this value (days) will be automatically deleted. +trace_ttl = None [keyvalue] # Allow encryption of values in key value stored qualified as "secret". diff --git a/st2common/bin/st2-purge-rule-enforcement b/st2common/bin/st2-purge-rule-enforcement new file mode 100755 index 0000000000..ac061dc626 --- /dev/null +++ b/st2common/bin/st2-purge-rule-enforcement @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from st2common.cmd.purge_rule_enforcement import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/st2common/bin/st2-purge-trace b/st2common/bin/st2-purge-trace new file mode 100755 index 0000000000..28343ecc33 --- /dev/null +++ b/st2common/bin/st2-purge-trace @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from st2common.cmd.purge_trace import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/st2common/setup.py b/st2common/setup.py index 53e0b07bef..ef4ba6a727 100644 --- a/st2common/setup.py +++ b/st2common/setup.py @@ -53,6 +53,8 @@ "bin/st2-register-content", "bin/st2-purge-executions", "bin/st2-purge-trigger-instances", + "bin/st2-purge-trace", + "bin/st2-purge-rule-enforcement", "bin/st2-run-pack-tests", "bin/st2ctl", "bin/st2-generate-symmetric-crypto-key", diff --git a/st2common/st2common/cmd/purge_rule_enforcement.py b/st2common/st2common/cmd/purge_rule_enforcement.py new file mode 100755 index 0000000000..22f37632be --- /dev/null +++ b/st2common/st2common/cmd/purge_rule_enforcement.py @@ -0,0 +1,81 @@ +# Copyright 2022 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +A utility script that purges trigger instances older than certain +timestamp. + +*** RISK RISK RISK. You will lose data. Run at your own risk. *** +""" + +from __future__ import absolute_import + +from datetime import datetime + +import six +import pytz +from oslo_config import cfg + +from st2common import config +from st2common import log as logging +from st2common.config import do_register_cli_opts +from st2common.script_setup import setup as common_setup +from st2common.script_setup import teardown as common_teardown +from st2common.constants.exit_codes import SUCCESS_EXIT_CODE +from st2common.constants.exit_codes import FAILURE_EXIT_CODE +from st2common.garbage_collection.rule_enforcement import purge_rule_enforcement + +__all__ = ["main"] + +LOG = logging.getLogger(__name__) + + +def _register_cli_opts(): + cli_opts = [ + cfg.StrOpt( + "timestamp", + default=None, + help="Will delete rule_enforcement instances older than " + + "this UTC timestamp. " + + "Example value: 2015-03-13T19:01:27.255542Z", + ) + ] + do_register_cli_opts(cli_opts) + + +def main(): + _register_cli_opts() + common_setup(config=config, setup_db=True, register_mq_exchanges=False) + + # Get config values + timestamp = cfg.CONF.timestamp + + if not timestamp: + LOG.error("Please supply a timestamp for purging models. Aborting.") + return 1 + else: + timestamp = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ") + timestamp = timestamp.replace(tzinfo=pytz.UTC) + + # Purge models. + try: + purge_rule_enforcement(logger=LOG, timestamp=timestamp) + except Exception as e: + LOG.exception(six.text_type(e)) + return FAILURE_EXIT_CODE + finally: + common_teardown() + + return SUCCESS_EXIT_CODE diff --git a/st2common/st2common/cmd/purge_trace.py b/st2common/st2common/cmd/purge_trace.py new file mode 100755 index 0000000000..db25ff8b76 --- /dev/null +++ b/st2common/st2common/cmd/purge_trace.py @@ -0,0 +1,81 @@ +# Copyright 2022 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +A utility script that purges trigger instances older than certain +timestamp. + +*** RISK RISK RISK. You will lose data. Run at your own risk. *** +""" + +from __future__ import absolute_import + +from datetime import datetime + +import six +import pytz +from oslo_config import cfg + +from st2common import config +from st2common import log as logging +from st2common.config import do_register_cli_opts +from st2common.script_setup import setup as common_setup +from st2common.script_setup import teardown as common_teardown +from st2common.constants.exit_codes import SUCCESS_EXIT_CODE +from st2common.constants.exit_codes import FAILURE_EXIT_CODE +from st2common.garbage_collection.trace import purge_trace + +__all__ = ["main"] + +LOG = logging.getLogger(__name__) + + +def _register_cli_opts(): + cli_opts = [ + cfg.StrOpt( + "timestamp", + default=None, + help="Will delete trace instances older than " + + "this UTC timestamp. " + + "Example value: 2015-03-13T19:01:27.255542Z", + ) + ] + do_register_cli_opts(cli_opts) + + +def main(): + _register_cli_opts() + common_setup(config=config, setup_db=True, register_mq_exchanges=False) + + # Get config values + timestamp = cfg.CONF.timestamp + + if not timestamp: + LOG.error("Please supply a timestamp for purging models. Aborting.") + return 1 + else: + timestamp = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ") + timestamp = timestamp.replace(tzinfo=pytz.UTC) + + # Purge models. + try: + purge_trace(logger=LOG, timestamp=timestamp) + except Exception as e: + LOG.exception(six.text_type(e)) + return FAILURE_EXIT_CODE + finally: + common_teardown() + + return SUCCESS_EXIT_CODE diff --git a/st2common/st2common/garbage_collection/rule_enforcement.py b/st2common/st2common/garbage_collection/rule_enforcement.py new file mode 100644 index 0000000000..bf986a771d --- /dev/null +++ b/st2common/st2common/garbage_collection/rule_enforcement.py @@ -0,0 +1,67 @@ +# Copyright 2022 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module with utility functions for purging old trigger instance objects. +""" + +from __future__ import absolute_import + +import six +from mongoengine.errors import InvalidQueryError + +from st2common.persistence.rule_enforcement import RuleEnforcement +from st2common.util import isotime + +__all__ = ["purge_rule_enforcement"] + + +def purge_rule_enforcement(logger, timestamp): + """ + :param timestamp: Rule enforcement instances older than this timestamp will be deleted. + :type timestamp: ``datetime.datetime + """ + if not timestamp: + raise ValueError("Specify a valid timestamp to purge.") + + logger.info( + "Purging rule enforcements older than timestamp: %s" + % timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + ) + + query_filters = {"enforced_at__lt": isotime.parse(timestamp)} + + try: + deleted_count = RuleEnforcement.delete_by_query(**query_filters) + except InvalidQueryError as e: + msg = ( + "Bad query (%s) used to delete rule enforcements: %s" + "Please contact support." + % ( + query_filters, + six.text_type(e), + ) + ) + raise InvalidQueryError(msg) + except: + logger.exception( + "Deleting rule enforcements using query_filters %s failed.", query_filters + ) + else: + logger.info("Deleted %s rule enforcement objects" % (deleted_count)) + + # Print stats + logger.info( + "All rule enforcement models older than timestamp %s were deleted.", timestamp + ) diff --git a/st2common/st2common/garbage_collection/trace.py b/st2common/st2common/garbage_collection/trace.py new file mode 100644 index 0000000000..a3109b3a56 --- /dev/null +++ b/st2common/st2common/garbage_collection/trace.py @@ -0,0 +1,65 @@ +# Copyright 2022 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module with utility functions for purging old trigger instance objects. +""" + +from __future__ import absolute_import + +import six +from mongoengine.errors import InvalidQueryError + +from st2common.persistence.trace import Trace +from st2common.util import isotime + +__all__ = ["purge_trace"] + + +def purge_trace(logger, timestamp): + """ + :param timestamp: Trace instances older than this timestamp will be deleted. + :type timestamp: ``datetime.datetime + """ + if not timestamp: + raise ValueError("Specify a valid timestamp to purge.") + + logger.info( + "Purging trace instances older than timestamp: %s" + % timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + ) + + query_filters = {"start_timestamp__lt": isotime.parse(timestamp)} + + try: + deleted_count = Trace.delete_by_query(**query_filters) + except InvalidQueryError as e: + msg = ( + "Bad query (%s) used to delete trace instances: %s" + "Please contact support." + % ( + query_filters, + six.text_type(e), + ) + ) + raise InvalidQueryError(msg) + except: + logger.exception( + "Deleting trace instances using query_filters %s failed.", query_filters + ) + else: + logger.info("Deleted %s trace objects" % (deleted_count)) + + # Print stats + logger.info("All trace models older than timestamp %s were deleted.", timestamp) diff --git a/st2common/st2common/persistence/rule_enforcement.py b/st2common/st2common/persistence/rule_enforcement.py index ce3b87d92f..7aa2826a91 100644 --- a/st2common/st2common/persistence/rule_enforcement.py +++ b/st2common/st2common/persistence/rule_enforcement.py @@ -24,3 +24,7 @@ class RuleEnforcement(Access): @classmethod def _get_impl(cls): return cls.impl + + @classmethod + def delete_by_query(cls, *args, **query): + return cls._get_impl().delete_by_query(*args, **query) diff --git a/st2common/st2common/persistence/trace.py b/st2common/st2common/persistence/trace.py index ce5472f2aa..9dca47e6b7 100644 --- a/st2common/st2common/persistence/trace.py +++ b/st2common/st2common/persistence/trace.py @@ -51,3 +51,7 @@ def push_rule(cls, instance, rule): @classmethod def push_trigger_instance(cls, instance, trigger_instance): return cls.update(instance, push__trigger_instances=trigger_instance) + + @classmethod + def delete_by_query(cls, *args, **query): + return cls._get_impl().delete_by_query(*args, **query) diff --git a/st2common/tests/unit/test_purge_rule_enforcement.py b/st2common/tests/unit/test_purge_rule_enforcement.py new file mode 100644 index 0000000000..b173dacbbe --- /dev/null +++ b/st2common/tests/unit/test_purge_rule_enforcement.py @@ -0,0 +1,77 @@ +# Copyright 2022 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from datetime import timedelta +import bson + +from st2common import log as logging +from st2common.garbage_collection.rule_enforcement import purge_rule_enforcement +from st2common.models.db.rule_enforcement import RuleEnforcementDB +from st2common.persistence.rule_enforcement import RuleEnforcement +from st2common.util import date as date_utils +from st2tests.base import CleanDbTestCase + +LOG = logging.getLogger(__name__) + + +class TestPurgeRuleEnforcement(CleanDbTestCase): + @classmethod + def setUpClass(cls): + CleanDbTestCase.setUpClass() + super(TestPurgeRuleEnforcement, cls).setUpClass() + + def setUp(self): + super(TestPurgeRuleEnforcement, self).setUp() + + def test_no_timestamp_doesnt_delete(self): + now = date_utils.get_datetime_utc_now() + saved = TestPurgeRuleEnforcement._create_save_rule_enforcement( + enforced_at=now - timedelta(days=20), + ) + + self.assertEqual(len(RuleEnforcement.get_all()), 1) + expected_msg = "Specify a valid timestamp" + self.assertRaisesRegexp( + ValueError, + expected_msg, + purge_rule_enforcement, + logger=LOG, + timestamp=None, + ) + self.assertEqual(len(RuleEnforcement.get_all()), 1) + + def test_purge(self): + now = date_utils.get_datetime_utc_now() + saved = TestPurgeRuleEnforcement._create_save_rule_enforcement( + enforced_at=now - timedelta(days=20), + ) + + saved = TestPurgeRuleEnforcement._create_save_rule_enforcement( + enforced_at=now - timedelta(days=5), + ) + + self.assertEqual(len(RuleEnforcement.get_all()), 2) + purge_rule_enforcement(logger=LOG, timestamp=now - timedelta(days=10)) + self.assertEqual(len(RuleEnforcement.get_all()), 1) + + @staticmethod + def _create_save_rule_enforcement(enforced_at): + created = RuleEnforcementDB( + trigger_instance_id=str(bson.ObjectId()), + rule={"ref": "foo_pack.foo_rule", "uid": "rule:foo_pack:foo_rule"}, + execution_id=str(bson.ObjectId()), + enforced_at=enforced_at, + ) + return RuleEnforcement.add_or_update(created) diff --git a/st2common/tests/unit/test_purge_trace.py b/st2common/tests/unit/test_purge_trace.py new file mode 100644 index 0000000000..386e3bab32 --- /dev/null +++ b/st2common/tests/unit/test_purge_trace.py @@ -0,0 +1,117 @@ +# Copyright 2022 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from datetime import timedelta +import bson + +from st2common import log as logging +from st2common.garbage_collection.trace import purge_trace +from st2common.models.db.trace import TraceDB, TraceComponentDB +from st2common.persistence.trace import Trace +from st2common.util import date as date_utils +from st2tests.base import CleanDbTestCase + +LOG = logging.getLogger(__name__) + + +class TestPurgeTrace(CleanDbTestCase): + @classmethod + def setUpClass(cls): + CleanDbTestCase.setUpClass() + super(TestPurgeTrace, cls).setUpClass() + + def setUp(self): + super(TestPurgeTrace, self).setUp() + + def test_no_timestamp_doesnt_delete(self): + now = date_utils.get_datetime_utc_now() + saved = TestPurgeTrace._create_save_trace( + trace_tag="test_trace", + action_executions=[str(bson.ObjectId()) for _ in range(4)], + rules=[str(bson.ObjectId()) for _ in range(4)], + trigger_instances=[str(bson.ObjectId()) for _ in range(5)], + start_timestamp=now - timedelta(days=20), + ) + + self.assertEqual(len(Trace.get_all()), 1) + expected_msg = "Specify a valid timestamp" + self.assertRaisesRegexp( + ValueError, + expected_msg, + purge_trace, + logger=LOG, + timestamp=None, + ) + self.assertEqual(len(Trace.get_all()), 1) + + def test_purge(self): + now = date_utils.get_datetime_utc_now() + saved = TestPurgeTrace._create_save_trace( + trace_tag="test_trace", + action_executions=[str(bson.ObjectId()) for _ in range(4)], + rules=[str(bson.ObjectId()) for _ in range(4)], + trigger_instances=[str(bson.ObjectId()) for _ in range(5)], + start_timestamp=now - timedelta(days=20), + ) + + saved = TestPurgeTrace._create_save_trace( + trace_tag="test_trace", + action_executions=[str(bson.ObjectId()) for _ in range(4)], + rules=[str(bson.ObjectId()) for _ in range(4)], + trigger_instances=[str(bson.ObjectId()) for _ in range(5)], + start_timestamp=now - timedelta(days=5), + ) + + self.assertEqual(len(Trace.get_all()), 2) + purge_trace(logger=LOG, timestamp=now - timedelta(days=10)) + self.assertEqual(len(Trace.get_all()), 1) + + @staticmethod + def _create_save_trace( + trace_tag, + id_=None, + action_executions=None, + rules=None, + trigger_instances=None, + start_timestamp=None, + ): + + if action_executions is None: + action_executions = [] + action_executions = [ + TraceComponentDB(object_id=action_execution) + for action_execution in action_executions + ] + + if rules is None: + rules = [] + rules = [TraceComponentDB(object_id=rule) for rule in rules] + + if trigger_instances is None: + trigger_instances = [] + trigger_instances = [ + TraceComponentDB(object_id=trigger_instance) + for trigger_instance in trigger_instances + ] + + created = TraceDB( + id=id_, + trace_tag=trace_tag, + trigger_instances=trigger_instances, + rules=rules, + action_executions=action_executions, + start_timestamp=start_timestamp, + ) + return Trace.add_or_update(created) diff --git a/st2reactor/st2reactor/garbage_collector/base.py b/st2reactor/st2reactor/garbage_collector/base.py index 49b32b449f..932c043565 100644 --- a/st2reactor/st2reactor/garbage_collector/base.py +++ b/st2reactor/st2reactor/garbage_collector/base.py @@ -41,6 +41,8 @@ from st2common.garbage_collection.executions import purge_orphaned_workflow_executions from st2common.garbage_collection.inquiries import purge_inquiries from st2common.garbage_collection.trigger_instances import purge_trigger_instances +from st2common.garbage_collection.trace import purge_trace +from st2common.garbage_collection.rule_enforcement import purge_rule_enforcement __all__ = ["GarbageCollectorService"] @@ -69,6 +71,8 @@ def __init__( cfg.CONF.garbagecollector.action_executions_output_ttl ) self._trigger_instances_ttl = cfg.CONF.garbagecollector.trigger_instances_ttl + self._trace_ttl = cfg.CONF.garbagecollector.trace_ttl + self._rule_enforcement_ttl = cfg.CONF.garbagecollector.rule_enforcement_ttl self._purge_inquiries = cfg.CONF.garbagecollector.purge_inquiries self._workflow_execution_max_idle = cfg.CONF.workflow_engine.gc_max_idle_sec @@ -153,6 +157,16 @@ def _validate_ttl_values(self): ) % (MINIMUM_TTL_DAYS_EXECUTION_OUTPUT) ) + if self._trace_ttl and self._trace_ttl < MINIMUM_TTL_DAYS: + raise ValueError( + "Minimum possible TTL for trace_ttl in days is %s" % (MINIMUM_TTL_DAYS) + ) + + if self._rule_enforcement_ttl and self._rule_enforcement_ttl < MINIMUM_TTL_DAYS: + raise ValueError( + "Minimum possible TTL for rule_enforcement_ttl in days is %s" + % (MINIMUM_TTL_DAYS) + ) def _perform_garbage_collection(self): LOG.info("Performing garbage collection...") @@ -198,6 +212,27 @@ def _perform_garbage_collection(self): else: LOG.debug(skip_message, obj_type) + obj_type = "trace" + + if self._trace_ttl and self._trace_ttl >= MINIMUM_TTL_DAYS: + LOG.info(proc_message, obj_type) + self._purge_trace() + concurrency.sleep(self._sleep_delay) + else: + LOG.debug(skip_message, obj_type) + + obj_type = "rule enforcement" + + if ( + self._rule_enforcement_ttl + and self._rule_enforcement_ttl >= MINIMUM_TTL_DAYS + ): + LOG.info(proc_message, obj_type) + self._purge_rule_enforcement() + concurrency.sleep(self._sleep_delay) + else: + LOG.debug(skip_message, obj_type) + obj_type = "inquiries" if self._purge_inquiries: LOG.info(proc_message, obj_type) @@ -307,6 +342,64 @@ def _purge_trigger_instances(self): return True + def _purge_trace(self): + """ + Purge trace objects which match the criteria defined in the config. + """ + utc_now = get_datetime_utc_now() + timestamp = utc_now - datetime.timedelta(days=self._trace_ttl) + + # Another sanity check to make sure we don't delete new executions + if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)): + raise ValueError( + "Calculated timestamp would violate the minimum TTL constraint" + ) + + timestamp_str = isotime.format(dt=timestamp) + LOG.info("Deleting trace objects older than: %s" % (timestamp_str)) + + if timestamp >= utc_now: + raise ValueError( + f"Calculated timestamp ({timestamp}) is" + f" later than now in UTC ({utc_now})." + ) + + try: + purge_trace(logger=LOG, timestamp=timestamp) + except Exception as e: + LOG.exception("Failed to delete trace: %s" % (six.text_type(e))) + + return True + + def _purge_rule_enforcement(self): + """ + Purge rule enforcements which match the criteria defined in the config. + """ + utc_now = get_datetime_utc_now() + timestamp = utc_now - datetime.timedelta(days=self._rule_enforcement_ttl) + + # Another sanity check to make sure we don't delete new executions + if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)): + raise ValueError( + "Calculated timestamp would violate the minimum TTL constraint" + ) + + timestamp_str = isotime.format(dt=timestamp) + LOG.info("Deleting rule enforcements older than: %s" % (timestamp_str)) + + if timestamp >= utc_now: + raise ValueError( + f"Calculated timestamp ({timestamp}) is" + f" later than now in UTC ({utc_now})." + ) + + try: + purge_executions(logger=LOG, timestamp=timestamp) + except Exception as e: + LOG.exception("Failed to delete rule enforcements: %s" % (six.text_type(e))) + + return True + def _timeout_inquiries(self): """Mark Inquiries as "timeout" that have exceeded their TTL""" try: diff --git a/st2reactor/st2reactor/garbage_collector/config.py b/st2reactor/st2reactor/garbage_collector/config.py index 4c6bb67255..01c1cee117 100644 --- a/st2reactor/st2reactor/garbage_collector/config.py +++ b/st2reactor/st2reactor/garbage_collector/config.py @@ -96,6 +96,16 @@ def _register_garbage_collector_opts(ignore_errors=False): default=None, help="Trigger instances older than this value (days) will be automatically deleted.", ), + cfg.IntOpt( + "rule_enforcement_ttl", + default=None, + help="Rule enforcements older than this value (days) will be automatically deleted.", + ), + cfg.IntOpt( + "trace_ttl", + default=None, + help="Trace objects older than this value (days) will be automatically deleted.", + ), ] common_config.do_register_opts( From 8804acbc5f0830ef9c57cc90aed2f5d949bcd46a Mon Sep 17 00:00:00 2001 From: Amanda McGuinness Date: Mon, 28 Mar 2022 11:29:04 +0000 Subject: [PATCH 2/2] Fix formatting and bug in purge rule from GC --- CHANGELOG.rst | 2 +- conf/st2.conf.sample | 8 ++++---- st2common/tests/unit/test_purge_rule_enforcement.py | 6 +++--- st2common/tests/unit/test_purge_trace.py | 6 +++--- st2reactor/st2reactor/garbage_collector/base.py | 6 +++--- st2tests/st2tests/config.py | 10 ++++++++++ 6 files changed, 24 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ce435f7f93..7c271c570f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -152,7 +152,7 @@ Added Contributed by @Kami. -* Added garbage collection for rule_enforcement and trace models +* Added garbage collection for rule_enforcement and trace models #5596 Contributed by Amanda McGuinness (@amanda11 intive) diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index 60c8648bc2..518562f5c9 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -177,14 +177,14 @@ collection_interval = 600 logging = /etc/st2/logging.garbagecollector.conf # Set to True to perform garbage collection on Inquiries (based on the TTL value per Inquiry) purge_inquiries = False +# Rule enforcements older than this value (days) will be automatically deleted. +rule_enforcement_ttl = None # How long to wait / sleep (in seconds) between collection of different object types. sleep_delay = 2 +# Trace objects older than this value (days) will be automatically deleted. +trace_ttl = None # Trigger instances older than this value (days) will be automatically deleted. trigger_instances_ttl = None -# Rule enforcement instances older than this value (days) will be automatically deleted. -rule_enforcement_ttl = None -# Trace instances older than this value (days) will be automatically deleted. -trace_ttl = None [keyvalue] # Allow encryption of values in key value stored qualified as "secret". diff --git a/st2common/tests/unit/test_purge_rule_enforcement.py b/st2common/tests/unit/test_purge_rule_enforcement.py index b173dacbbe..cdc20f132d 100644 --- a/st2common/tests/unit/test_purge_rule_enforcement.py +++ b/st2common/tests/unit/test_purge_rule_enforcement.py @@ -37,7 +37,7 @@ def setUp(self): def test_no_timestamp_doesnt_delete(self): now = date_utils.get_datetime_utc_now() - saved = TestPurgeRuleEnforcement._create_save_rule_enforcement( + TestPurgeRuleEnforcement._create_save_rule_enforcement( enforced_at=now - timedelta(days=20), ) @@ -54,11 +54,11 @@ def test_no_timestamp_doesnt_delete(self): def test_purge(self): now = date_utils.get_datetime_utc_now() - saved = TestPurgeRuleEnforcement._create_save_rule_enforcement( + TestPurgeRuleEnforcement._create_save_rule_enforcement( enforced_at=now - timedelta(days=20), ) - saved = TestPurgeRuleEnforcement._create_save_rule_enforcement( + TestPurgeRuleEnforcement._create_save_rule_enforcement( enforced_at=now - timedelta(days=5), ) diff --git a/st2common/tests/unit/test_purge_trace.py b/st2common/tests/unit/test_purge_trace.py index 386e3bab32..5ce472fa35 100644 --- a/st2common/tests/unit/test_purge_trace.py +++ b/st2common/tests/unit/test_purge_trace.py @@ -37,7 +37,7 @@ def setUp(self): def test_no_timestamp_doesnt_delete(self): now = date_utils.get_datetime_utc_now() - saved = TestPurgeTrace._create_save_trace( + TestPurgeTrace._create_save_trace( trace_tag="test_trace", action_executions=[str(bson.ObjectId()) for _ in range(4)], rules=[str(bson.ObjectId()) for _ in range(4)], @@ -58,7 +58,7 @@ def test_no_timestamp_doesnt_delete(self): def test_purge(self): now = date_utils.get_datetime_utc_now() - saved = TestPurgeTrace._create_save_trace( + TestPurgeTrace._create_save_trace( trace_tag="test_trace", action_executions=[str(bson.ObjectId()) for _ in range(4)], rules=[str(bson.ObjectId()) for _ in range(4)], @@ -66,7 +66,7 @@ def test_purge(self): start_timestamp=now - timedelta(days=20), ) - saved = TestPurgeTrace._create_save_trace( + TestPurgeTrace._create_save_trace( trace_tag="test_trace", action_executions=[str(bson.ObjectId()) for _ in range(4)], rules=[str(bson.ObjectId()) for _ in range(4)], diff --git a/st2reactor/st2reactor/garbage_collector/base.py b/st2reactor/st2reactor/garbage_collector/base.py index 932c043565..e0a651e7cb 100644 --- a/st2reactor/st2reactor/garbage_collector/base.py +++ b/st2reactor/st2reactor/garbage_collector/base.py @@ -349,7 +349,7 @@ def _purge_trace(self): utc_now = get_datetime_utc_now() timestamp = utc_now - datetime.timedelta(days=self._trace_ttl) - # Another sanity check to make sure we don't delete new executions + # Another sanity check to make sure we don't delete new objects if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)): raise ValueError( "Calculated timestamp would violate the minimum TTL constraint" @@ -378,7 +378,7 @@ def _purge_rule_enforcement(self): utc_now = get_datetime_utc_now() timestamp = utc_now - datetime.timedelta(days=self._rule_enforcement_ttl) - # Another sanity check to make sure we don't delete new executions + # Another sanity check to make sure we don't delete new objects if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)): raise ValueError( "Calculated timestamp would violate the minimum TTL constraint" @@ -394,7 +394,7 @@ def _purge_rule_enforcement(self): ) try: - purge_executions(logger=LOG, timestamp=timestamp) + purge_rule_enforcement(logger=LOG, timestamp=timestamp) except Exception as e: LOG.exception("Failed to delete rule enforcements: %s" % (six.text_type(e))) diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py index c4ae0cfcd1..6ef73fd4ae 100644 --- a/st2tests/st2tests/config.py +++ b/st2tests/st2tests/config.py @@ -484,6 +484,16 @@ def _register_garbage_collector_opts(): default=None, help="Trigger instances older than this value (days) will be automatically deleted.", ), + cfg.IntOpt( + "rule_enforcement_ttl", + default=None, + help="Rule enforcements older than this value (days) will be automatically deleted.", + ), + cfg.IntOpt( + "trace_ttl", + default=None, + help="Trace objects older than this value (days) will be automatically deleted.", + ), ] _register_opts(ttl_opts, group="garbagecollector")