[WIP] [Issue#4939] Changes to support new purge task executions and purge workflows

conf/st2.conf.sample

@@ -165,6 +165,11 @@ purge_inquiries = False
 sleep_delay = 2
 # Trigger instances older than this value (days) will be automatically deleted.
 trigger_instances_ttl = None
+# Workflow execution output objects (ones generated by action output streaming) older than this value (days) will be automatically deleted.
+workflow_execution_ttl = 7
+# Task execution output objects (ones generated by action output streaming) older than this value (days) will be automatically deleted.
+task_execution_ttl = 7
+
 
 [keyvalue]
 # Allow encryption of values in key value stored qualified as "secret".

st2common/bin/st2-purge-task-executions

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from st2common.cmd.purge_task_executions import main
+
+if __name__ == '__main__':
+    sys.exit(main())

st2common/bin/st2-purge-workflows

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from st2common.cmd.purge_workflows import main
+
+if __name__ == '__main__':
+    sys.exit(main())

st2common/setup.py

@@ -50,6 +50,8 @@
         'bin/st2-cleanup-db',
         'bin/st2-register-content',
         'bin/st2-purge-executions',
+        'bin/st2-purge-workflows',
+        'bin/st2-purge-task-executions',
         'bin/st2-purge-trigger-instances',
         'bin/st2-run-pack-tests',
         'bin/st2ctl',

st2common/st2common/cmd/purge_task_executions.py

@@ -0,0 +1,88 @@
+# Copyright 2019 Extreme Networks, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+A utility script that purges st2 executions older than certain
+timestamp.
+
+*** RISK RISK RISK. You will lose data. Run at your own risk. ***
+"""
+
+from __future__ import absolute_import
+
+from datetime import datetime
+
+import six
+import pytz
+from oslo_config import cfg
+
+from st2common import config
+from st2common import log as logging
+from st2common.config import do_register_cli_opts
+from st2common.script_setup import setup as common_setup
+from st2common.script_setup import teardown as common_teardown
+from st2common.constants.exit_codes import SUCCESS_EXIT_CODE
+from st2common.constants.exit_codes import FAILURE_EXIT_CODE
+from st2common.garbage_collection.workflows import purge_task_execution
+
+__all__ = [
+    'main'
+]
+
+LOG = logging.getLogger(__name__)
+
+
+def _register_cli_opts():
+    cli_opts = [
+        cfg.StrOpt('timestamp', default=None,
+                   help='Will delete execution and liveaction models older than ' +
+                   'this UTC timestamp. ' +
+                   'Example value: 2015-03-13T19:01:27.255542Z.'),
+        cfg.StrOpt('action-ref', default='',
+                   help='action-ref to delete executions for.'),
+        cfg.BoolOpt('purge-incomplete', default=False,
+                    help='Purge all models irrespective of their ``status``.' +
+                    'By default, only executions in completed states such as "succeeeded" ' +
+                    ', "failed", "canceled" and "timed_out" are deleted.'),
+    ]
+    do_register_cli_opts(cli_opts)
+
+
+def main():
+    _register_cli_opts()
+    common_setup(config=config, setup_db=True, register_mq_exchanges=False)
+
+    # Get config values
+    timestamp = cfg.CONF.timestamp
+    action_ref = cfg.CONF.action_ref
+    purge_incomplete = cfg.CONF.purge_incomplete
+
+    if not timestamp:
+        LOG.error('Please supply a timestamp for purging models. Aborting.')
+        return 1
+    else:
+        timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ')
+        timestamp = timestamp.replace(tzinfo=pytz.UTC)
+
+    try:
+        purge_task_execution(logger=LOG, timestamp=timestamp, action_ref=action_ref,
+                             purge_incomplete=purge_incomplete)
+    except Exception as e:
+        LOG.exception(six.text_type(e))
+        return FAILURE_EXIT_CODE
+    finally:
+        common_teardown()
+
+    return SUCCESS_EXIT_CODE

st2common/st2common/cmd/purge_workflows.py

@@ -0,0 +1,88 @@
+# Copyright 2019 Extreme Networks, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+A utility script that purges st2 executions older than certain
+timestamp.
+
+*** RISK RISK RISK. You will lose data. Run at your own risk. ***
+"""
+
+from __future__ import absolute_import
+
+from datetime import datetime
+
+import six
+import pytz
+from oslo_config import cfg
+
+from st2common import config
+from st2common import log as logging
+from st2common.config import do_register_cli_opts
+from st2common.script_setup import setup as common_setup
+from st2common.script_setup import teardown as common_teardown
+from st2common.constants.exit_codes import SUCCESS_EXIT_CODE
+from st2common.constants.exit_codes import FAILURE_EXIT_CODE
+from st2common.garbage_collection.workflows import purge_workflow_execution
+
+__all__ = [
+    'main'
+]
+
+LOG = logging.getLogger(__name__)
+
+
+def _register_cli_opts():
+    cli_opts = [
+        cfg.StrOpt('timestamp', default=None,
+                   help='Will delete execution and liveaction models older than ' +
+                   'this UTC timestamp. ' +
+                   'Example value: 2015-03-13T19:01:27.255542Z.'),
+        cfg.StrOpt('action-ref', default='',
+                   help='action-ref to delete executions for.'),
+        cfg.BoolOpt('purge-incomplete', default=False,
+                    help='Purge all models irrespective of their ``status``.' +
+                    'By default, only executions in completed states such as "succeeeded" ' +
+                    ', "failed", "canceled" and "timed_out" are deleted.'),
+    ]
+    do_register_cli_opts(cli_opts)
+
+
+def main():
+    _register_cli_opts()
+    common_setup(config=config, setup_db=True, register_mq_exchanges=False)
+
+    # Get config values
+    timestamp = cfg.CONF.timestamp
+    action_ref = cfg.CONF.action_ref
+    purge_incomplete = cfg.CONF.purge_incomplete
+
+    if not timestamp:
+        LOG.error('Please supply a timestamp for purging models. Aborting.')
+        return 1
+    else:
+        timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ')
+        timestamp = timestamp.replace(tzinfo=pytz.UTC)
+
+    try:
+        purge_workflow_execution(logger=LOG, timestamp=timestamp, action_ref=action_ref,
+                                 purge_incomplete=purge_incomplete)
+    except Exception as e:
+        LOG.exception(six.text_type(e))
+        return FAILURE_EXIT_CODE
+    finally:
+        common_teardown()
+
+    return SUCCESS_EXIT_CODE