Add garbage collection for trace and rule_enforcement models

CHANGELOG.rst

@@ -152,6 +152,10 @@ Added
 
   Contributed by @Kami.
 
+* Added garbage collection for rule_enforcement and trace models #5596
+  Contributed by Amanda McGuinness (@amanda11 intive)
+
+
 Fixed
 ~~~~~
 

conf/st2.conf.sample

@@ -177,8 +177,12 @@ collection_interval = 600
 logging = /etc/st2/logging.garbagecollector.conf
 # Set to True to perform garbage collection on Inquiries (based on the TTL value per Inquiry)
 purge_inquiries = False
+# Rule enforcements older than this value (days) will be automatically deleted.
+rule_enforcement_ttl = None
 # How long to wait / sleep (in seconds) between collection of different object types.
 sleep_delay = 2
+# Trace objects older than this value (days) will be automatically deleted.
+trace_ttl = None
 # Trigger instances older than this value (days) will be automatically deleted.
 trigger_instances_ttl = None
 

st2common/bin/st2-purge-rule-enforcement

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from st2common.cmd.purge_rule_enforcement import main
+
+if __name__ == "__main__":
+    sys.exit(main())

st2common/bin/st2-purge-trace

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from st2common.cmd.purge_trace import main
+
+if __name__ == "__main__":
+    sys.exit(main())

st2common/setup.py

@@ -53,6 +53,8 @@
         "bin/st2-register-content",
         "bin/st2-purge-executions",
         "bin/st2-purge-trigger-instances",
+        "bin/st2-purge-trace",
+        "bin/st2-purge-rule-enforcement",
         "bin/st2-run-pack-tests",
         "bin/st2ctl",
         "bin/st2-generate-symmetric-crypto-key",

st2common/st2common/cmd/purge_rule_enforcement.py

@@ -0,0 +1,81 @@
+# Copyright 2022 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+A utility script that purges trigger instances older than certain
+timestamp.
+
+*** RISK RISK RISK. You will lose data. Run at your own risk. ***
+"""
+
+from __future__ import absolute_import
+
+from datetime import datetime
+
+import six
+import pytz
+from oslo_config import cfg
+
+from st2common import config
+from st2common import log as logging
+from st2common.config import do_register_cli_opts
+from st2common.script_setup import setup as common_setup
+from st2common.script_setup import teardown as common_teardown
+from st2common.constants.exit_codes import SUCCESS_EXIT_CODE
+from st2common.constants.exit_codes import FAILURE_EXIT_CODE
+from st2common.garbage_collection.rule_enforcement import purge_rule_enforcement
+
+__all__ = ["main"]
+
+LOG = logging.getLogger(__name__)
+
+
+def _register_cli_opts():
+    cli_opts = [
+        cfg.StrOpt(
+            "timestamp",
+            default=None,
+            help="Will delete rule_enforcement instances older than "
+            + "this UTC timestamp. "
+            + "Example value: 2015-03-13T19:01:27.255542Z",
+        )
+    ]
+    do_register_cli_opts(cli_opts)
+
+
+def main():
+    _register_cli_opts()
+    common_setup(config=config, setup_db=True, register_mq_exchanges=False)
+
+    # Get config values
+    timestamp = cfg.CONF.timestamp
+
+    if not timestamp:
+        LOG.error("Please supply a timestamp for purging models. Aborting.")
+        return 1
+    else:
+        timestamp = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ")
+        timestamp = timestamp.replace(tzinfo=pytz.UTC)
+
+    # Purge models.
+    try:
+        purge_rule_enforcement(logger=LOG, timestamp=timestamp)
+    except Exception as e:
+        LOG.exception(six.text_type(e))
+        return FAILURE_EXIT_CODE
+    finally:
+        common_teardown()
+
+    return SUCCESS_EXIT_CODE

st2common/st2common/cmd/purge_trace.py

@@ -0,0 +1,81 @@
+# Copyright 2022 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+A utility script that purges trigger instances older than certain
+timestamp.
+
+*** RISK RISK RISK. You will lose data. Run at your own risk. ***
+"""
+
+from __future__ import absolute_import
+
+from datetime import datetime
+
+import six
+import pytz
+from oslo_config import cfg
+
+from st2common import config
+from st2common import log as logging
+from st2common.config import do_register_cli_opts
+from st2common.script_setup import setup as common_setup
+from st2common.script_setup import teardown as common_teardown
+from st2common.constants.exit_codes import SUCCESS_EXIT_CODE
+from st2common.constants.exit_codes import FAILURE_EXIT_CODE
+from st2common.garbage_collection.trace import purge_trace
+
+__all__ = ["main"]
+
+LOG = logging.getLogger(__name__)
+
+
+def _register_cli_opts():
+    cli_opts = [
+        cfg.StrOpt(
+            "timestamp",
+            default=None,
+            help="Will delete trace instances older than "
+            + "this UTC timestamp. "
+            + "Example value: 2015-03-13T19:01:27.255542Z",
+        )
+    ]
+    do_register_cli_opts(cli_opts)
+
+
+def main():
+    _register_cli_opts()
+    common_setup(config=config, setup_db=True, register_mq_exchanges=False)
+
+    # Get config values
+    timestamp = cfg.CONF.timestamp
+
+    if not timestamp:
+        LOG.error("Please supply a timestamp for purging models. Aborting.")
+        return 1
+    else:
+        timestamp = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ")
+        timestamp = timestamp.replace(tzinfo=pytz.UTC)
+
+    # Purge models.
+    try:
+        purge_trace(logger=LOG, timestamp=timestamp)
+    except Exception as e:
+        LOG.exception(six.text_type(e))
+        return FAILURE_EXIT_CODE
+    finally:
+        common_teardown()
+
+    return SUCCESS_EXIT_CODE

st2common/st2common/garbage_collection/rule_enforcement.py

@@ -0,0 +1,67 @@
+# Copyright 2022 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Module with utility functions for purging old trigger instance objects.
+"""
+
+from __future__ import absolute_import
+
+import six
+from mongoengine.errors import InvalidQueryError
+
+from st2common.persistence.rule_enforcement import RuleEnforcement
+from st2common.util import isotime
+
+__all__ = ["purge_rule_enforcement"]
+
+
+def purge_rule_enforcement(logger, timestamp):
+    """
+    :param timestamp: Rule enforcement instances older than this timestamp will be deleted.
+    :type timestamp: ``datetime.datetime
+    """
+    if not timestamp:
+        raise ValueError("Specify a valid timestamp to purge.")
+
+    logger.info(
+        "Purging rule enforcements older than timestamp: %s"
+        % timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+    )
+
+    query_filters = {"enforced_at__lt": isotime.parse(timestamp)}
+
+    try:
+        deleted_count = RuleEnforcement.delete_by_query(**query_filters)
+    except InvalidQueryError as e:
+        msg = (
+            "Bad query (%s) used to delete rule enforcements: %s"
+            "Please contact support."
+            % (
+                query_filters,
+                six.text_type(e),
+            )
+        )
+        raise InvalidQueryError(msg)
+    except:
+        logger.exception(
+            "Deleting rule enforcements using query_filters %s failed.", query_filters
+        )
+    else:
+        logger.info("Deleted %s rule enforcement objects" % (deleted_count))
+
+    # Print stats
+    logger.info(
+        "All rule enforcement models older than timestamp %s were deleted.", timestamp
+    )

st2common/st2common/garbage_collection/trace.py

@@ -0,0 +1,65 @@
+# Copyright 2022 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Module with utility functions for purging old trigger instance objects.
+"""
+
+from __future__ import absolute_import
+
+import six
+from mongoengine.errors import InvalidQueryError
+
+from st2common.persistence.trace import Trace
+from st2common.util import isotime
+
+__all__ = ["purge_trace"]
+
+
+def purge_trace(logger, timestamp):
+    """
+    :param timestamp: Trace instances older than this timestamp will be deleted.
+    :type timestamp: ``datetime.datetime
+    """
+    if not timestamp:
+        raise ValueError("Specify a valid timestamp to purge.")
+
+    logger.info(
+        "Purging trace instances older than timestamp: %s"
+        % timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+    )
+
+    query_filters = {"start_timestamp__lt": isotime.parse(timestamp)}
+
+    try:
+        deleted_count = Trace.delete_by_query(**query_filters)
+    except InvalidQueryError as e:
+        msg = (
+            "Bad query (%s) used to delete trace instances: %s"
+            "Please contact support."
+            % (
+                query_filters,
+                six.text_type(e),
+            )
+        )
+        raise InvalidQueryError(msg)
+    except:
+        logger.exception(
+            "Deleting trace instances using query_filters %s failed.", query_filters
+        )
+    else:
+        logger.info("Deleted %s trace objects" % (deleted_count))
+
+    # Print stats
+    logger.info("All trace models older than timestamp %s were deleted.", timestamp)

st2common/st2common/persistence/rule_enforcement.py

@@ -24,3 +24,7 @@ class RuleEnforcement(Access):
     @classmethod
     def _get_impl(cls):
         return cls.impl
+
+    @classmethod
+    def delete_by_query(cls, *args, **query):
+        return cls._get_impl().delete_by_query(*args, **query)

st2common/st2common/persistence/trace.py

@@ -51,3 +51,7 @@ def push_rule(cls, instance, rule):
     @classmethod
     def push_trigger_instance(cls, instance, trigger_instance):
         return cls.update(instance, push__trigger_instances=trigger_instance)
+
+    @classmethod
+    def delete_by_query(cls, *args, **query):
+        return cls._get_impl().delete_by_query(*args, **query)