From c826dbc74323acbd97e2fb39e415050cdb730715 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 18 Dec 2020 22:26:05 +0100 Subject: [PATCH 01/31] Remove st2resultstracker logging configuration --- conf/logrotate.conf | 9 ----- st2actions/conf/logging.resultstracker.conf | 44 --------------------- st2actions/conf/syslog.resultstracker.conf | 22 ----------- 3 files changed, 75 deletions(-) delete mode 100644 st2actions/conf/logging.resultstracker.conf delete mode 100644 st2actions/conf/syslog.resultstracker.conf diff --git a/conf/logrotate.conf b/conf/logrotate.conf index 7940fead0e..c5f60005d6 100644 --- a/conf/logrotate.conf +++ b/conf/logrotate.conf @@ -71,15 +71,6 @@ notifempty endscript } -## Results Tracker -/var/log/st2/st2resultstracker*.log { - daily - rotate 5 - postrotate - st2ctl reopen-log-files st2resultstracker > /dev/null - endscript -} - ## Notifier /var/log/st2/st2notifier*.log { daily diff --git a/st2actions/conf/logging.resultstracker.conf b/st2actions/conf/logging.resultstracker.conf deleted file mode 100644 index 67551477ff..0000000000 --- a/st2actions/conf/logging.resultstracker.conf +++ /dev/null @@ -1,44 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=consoleHandler, fileHandler, auditHandler - -[formatters] -keys=simpleConsoleFormatter, verboseConsoleFormatter, gelfFormatter - -[logger_root] -level=DEBUG -handlers=consoleHandler, fileHandler, auditHandler - -[handler_consoleHandler] -class=StreamHandler -level=INFO -formatter=simpleConsoleFormatter -args=(sys.stdout,) - -[handler_fileHandler] -class=st2common.log.FormatNamedFileHandler -level=DEBUG -formatter=verboseConsoleFormatter -args=('logs/st2resultstracker.log',) - -[handler_auditHandler] -class=st2common.log.FormatNamedFileHandler -level=AUDIT -formatter=gelfFormatter -args=('logs/st2resultstracker.audit.log',) - -[formatter_simpleConsoleFormatter] -class=st2common.logging.formatters.ConsoleLogFormatter -format=%(asctime)s %(levelname)s [-] %(message)s -datefmt= - -[formatter_verboseConsoleFormatter] -class=st2common.logging.formatters.ConsoleLogFormatter -format=%(asctime)s %(thread)s %(levelname)s %(module)s [-] %(message)s -datefmt= - -[formatter_gelfFormatter] -class=st2common.logging.formatters.GelfLogFormatter -format=%(message)s diff --git a/st2actions/conf/syslog.resultstracker.conf b/st2actions/conf/syslog.resultstracker.conf deleted file mode 100644 index 2196a44377..0000000000 --- a/st2actions/conf/syslog.resultstracker.conf +++ /dev/null @@ -1,22 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=syslogHandler - -[formatters] -keys=syslogVerboseFormatter - -[logger_root] -level=DEBUG -handlers=syslogHandler - -[handler_syslogHandler] -class=st2common.log.ConfigurableSyslogHandler -level=DEBUG -formatter=syslogVerboseFormatter -args=() - -[formatter_syslogVerboseFormatter] -format=st2resultstracker[%(process)d]: %(levelname)s %(thread)s %(module)s [-] %(message)s -datefmt= From dd8f6fef4a4b3dc3b78b638614edf090eba49416 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 18 Dec 2020 22:27:05 +0100 Subject: [PATCH 02/31] Remove st2resultstracker from the st2actions setup.py --- st2actions/bin/st2resultstracker | 22 ---------------------- st2actions/setup.py | 1 - 2 files changed, 23 deletions(-) delete mode 100755 st2actions/bin/st2resultstracker diff --git a/st2actions/bin/st2resultstracker b/st2actions/bin/st2resultstracker deleted file mode 100755 index 0dae3ad493..0000000000 --- a/st2actions/bin/st2resultstracker +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the StackStorm, Inc ('StackStorm') under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -from st2actions.cmd import st2resultstracker - - -if __name__ == '__main__': - sys.exit(st2resultstracker.main()) diff --git a/st2actions/setup.py b/st2actions/setup.py index 4845eb83d9..6fcb2cde92 100644 --- a/st2actions/setup.py +++ b/st2actions/setup.py @@ -47,7 +47,6 @@ scripts=[ 'bin/st2actionrunner', 'bin/st2notifier', - 'bin/st2resultstracker', 'bin/st2workflowengine', 'bin/st2scheduler', ] From 5f3a1ebc234491fe2f9f68eb8f2a81cc0026016c Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 18 Dec 2020 22:27:39 +0100 Subject: [PATCH 03/31] Remove st2resultstracker from st2ctl --- st2common/bin/st2ctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/bin/st2ctl b/st2common/bin/st2ctl index 29c71aae85..f2cd03ee22 100755 --- a/st2common/bin/st2ctl +++ b/st2common/bin/st2ctl @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -COMPONENTS="st2actionrunner st2api st2stream st2auth st2garbagecollector st2notifier st2resultstracker st2rulesengine st2sensorcontainer st2chatops st2timersengine st2workflowengine st2scheduler" +COMPONENTS="st2actionrunner st2api st2stream st2auth st2garbagecollector st2notifier st2rulesengine st2sensorcontainer st2chatops st2timersengine st2workflowengine st2scheduler" ST2_CONF="/etc/st2/st2.conf" # Ensure global environment is sourced if exists From 5c2eea692743057a7dc922b03d7a4ed28296e7b0 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 18 Dec 2020 22:28:13 +0100 Subject: [PATCH 04/31] Remove st2resultstracker from the launchdev setup and screen configs --- tools/launchdev.sh | 6 ------ tools/screen-configs/st2resultstracker.conf | 6 ------ 2 files changed, 12 deletions(-) delete mode 100644 tools/screen-configs/st2resultstracker.conf diff --git a/tools/launchdev.sh b/tools/launchdev.sh index 20f63b4fb7..774b41067f 100755 --- a/tools/launchdev.sh +++ b/tools/launchdev.sh @@ -266,12 +266,6 @@ function st2start(){ ./st2reactor/bin/st2timersengine \ --config-file $ST2_CONF - # Run the results tracker - echo 'Starting screen session st2-resultstracker...' - screen -L -c tools/screen-configs/st2resultstracker.conf -d -m -S st2-resultstracker ${VIRTUALENV}/bin/python \ - ./st2actions/bin/st2resultstracker \ - --config-file $ST2_CONF - # Run the actions notifier echo 'Starting screen session st2-notifier...' screen -L -c tools/screen-configs/st2notifier.conf -d -m -S st2-notifier ${VIRTUALENV}/bin/python \ diff --git a/tools/screen-configs/st2resultstracker.conf b/tools/screen-configs/st2resultstracker.conf deleted file mode 100644 index c5d454d4d0..0000000000 --- a/tools/screen-configs/st2resultstracker.conf +++ /dev/null @@ -1,6 +0,0 @@ -logfile logs/screen-st2resultstracker.log -logfile flush 1 -log on -logtstamp after 1 -logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" -logtstamp on From 844f52767d6348b0eb3ea1037cbaa65daeb62ec0 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 18 Dec 2020 22:31:17 +0100 Subject: [PATCH 05/31] Add note about the st2resultstracker removal to the changelog --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d74348be48..15aa27d4bf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -49,10 +49,14 @@ Fixed Removed ~~~~~~~~ * Removed check-licence script (cleanup) #5092 + Contributed by @kroustou * Updated Makefile and CI to use Python 3 only, removing Python 2 (cleanup) #5090 Contributed by @blag +* Remove st2resultstracker from st2ctl, the development environment and the st2actions setup.py (cleanup) #5108 + + Contributed by @winem 3.3.0 - October 06, 2020 ------------------------ From 7bbd88b38cbc5bdd63e0a49696423e0f76d0cb13 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 18 Dec 2020 22:40:04 +0100 Subject: [PATCH 06/31] Remove the resultstracker from st2.tests2.conf --- conf/st2.tests2.conf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/conf/st2.tests2.conf b/conf/st2.tests2.conf index 1536789144..cb7be38a71 100644 --- a/conf/st2.tests2.conf +++ b/conf/st2.tests2.conf @@ -89,10 +89,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf From ce6f57238573ee26fe220d2978035e638d3be194 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 18 Dec 2020 22:40:19 +0100 Subject: [PATCH 07/31] Remove the resultstracker from st2.conf.sample --- conf/st2.conf.sample | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index 6e0abb00ca..9a5b92eb26 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -228,18 +228,6 @@ logging = /etc/st2/logging.notifier.conf # Enable/Disable support for pack common libs. Setting this config to ``True`` would allow you to place common library code for sensors and actions in lib/ folder in packs and use them in python sensors and actions. See https://docs.stackstorm.com/reference/sharing_code_sensors_actions.html for details. enable_common_libs = False -[resultstracker] -# Sleep delay in between queries when query queue is empty. -empty_q_sleep_time = 1 -# Location of the logging configuration file. -logging = /etc/st2/logging.resultstracker.conf -# Sleep delay for query when there is no more worker in pool. -no_workers_sleep_time = 1 -# Time interval between queries to external workflow system. -query_interval = 5 -# Number of threads to use to query external workflow systems. -thread_pool_size = 10 - [rulesengine] # Location of the logging configuration file. logging = /etc/st2/logging.rulesengine.conf From 446f7f9ac6a10d17a3f6350c93f4f2b2e038f9d3 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 18 Dec 2020 22:41:40 +0100 Subject: [PATCH 08/31] Remove the resultstracker from st2.dev.conf --- conf/st2.dev.conf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/conf/st2.dev.conf b/conf/st2.dev.conf index 08aa45870d..2357b08263 100644 --- a/conf/st2.dev.conf +++ b/conf/st2.dev.conf @@ -114,10 +114,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [scheduler] logging = st2actions/conf/logging.scheduler.conf From e1a8dc713b60f87f91664b8b6ea61b3ef813263f Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 18 Dec 2020 22:41:57 +0100 Subject: [PATCH 09/31] Remove the resultstracker from st2.package.conf --- conf/st2.package.conf | 3 --- 1 file changed, 3 deletions(-) diff --git a/conf/st2.package.conf b/conf/st2.package.conf index 661d877834..dbd10b211b 100644 --- a/conf/st2.package.conf +++ b/conf/st2.package.conf @@ -22,9 +22,6 @@ logging = /etc/st2/logging.rulesengine.conf logging = /etc/st2/logging.actionrunner.conf virtualenv_opts = --always-copy -[resultstracker] -logging = /etc/st2/logging.resultstracker.conf - [notifier] logging = /etc/st2/logging.notifier.conf From fad451f1acbd44a86cf4d3fb3aabb66217b25e97 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 18 Dec 2020 22:42:40 +0100 Subject: [PATCH 10/31] Remove the resultstracker from st2 test configurations --- conf/st2.tests.conf | 4 ---- conf/st2.tests1.conf | 3 --- 2 files changed, 7 deletions(-) diff --git a/conf/st2.tests.conf b/conf/st2.tests.conf index 87c97595e4..0e9afeb288 100644 --- a/conf/st2.tests.conf +++ b/conf/st2.tests.conf @@ -84,10 +84,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/conf/st2.tests1.conf b/conf/st2.tests1.conf index 2ec4592807..53ae62e5ae 100644 --- a/conf/st2.tests1.conf +++ b/conf/st2.tests1.conf @@ -67,9 +67,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf - [notifier] logging = st2actions/conf/logging.notifier.conf From 8c3ae077b40fa1ed195b5ba12a417d0b356dbb67 Mon Sep 17 00:00:00 2001 From: Eugen Cusmaunsa Date: Mon, 21 Dec 2020 14:35:58 +0000 Subject: [PATCH 11/31] Update CHANGELOG.rst --- CHANGELOG.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1a65350883..357cbb89d7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,7 +6,6 @@ in development Added ~~~~~ - * Added st2-rbac-backend pip requirements for RBAC integration. (new feature) #5086 Contributed by @hnanchahal From 2237f781a5bad8fab36a8c29672310e4e6e1a70d Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 24 Dec 2020 19:27:35 +0100 Subject: [PATCH 12/31] Remove st2actions.resultstracker.config from the config_gen script --- tools/config_gen.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/config_gen.py b/tools/config_gen.py index 30769b3902..0f6215babf 100755 --- a/tools/config_gen.py +++ b/tools/config_gen.py @@ -27,7 +27,6 @@ CONFIGS = ['st2actions.config', 'st2actions.scheduler.config', 'st2actions.notifier.config', - 'st2actions.resultstracker.config', 'st2actions.workflows.config', 'st2api.config', 'st2stream.config', From 992023bfe14e35ac1066d5c574adb944a65a12d2 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 24 Dec 2020 19:28:02 +0100 Subject: [PATCH 13/31] Remove the resultstracker config options from the st2common config.py --- st2common/st2common/config.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index d02ded345d..c86b64ab47 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -472,25 +472,6 @@ def register_opts(ignore_errors=False): do_register_opts(coord_opts, 'coordination', ignore_errors) - # Results Tracker query module options - # Note that these are currently not used - query_opts = [ - cfg.IntOpt( - 'thread_pool_size', default=10, - help='Number of threads to use to query external workflow systems.'), - cfg.FloatOpt( - 'query_interval', default=5, - help='Time interval between queries to external workflow system.'), - cfg.FloatOpt( - 'empty_q_sleep_time', default=1, - help='Sleep delay in between queries when query queue is empty.'), - cfg.FloatOpt( - 'no_workers_sleep_time', default=1, - help='Sleep delay for query when there is no more worker in pool.') - ] - - do_register_opts(query_opts, group='resultstracker', ignore_errors=ignore_errors) - # XXX: This is required for us to support deprecated config group results_tracker query_opts = [ cfg.IntOpt( From a29d7700a85023c4165670a740397f2b28986891 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 24 Dec 2020 20:13:44 +0100 Subject: [PATCH 14/31] remove st2resultstracker screen for the dev env --- tools/launchdev.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/launchdev.sh b/tools/launchdev.sh index 774b41067f..9e6fc9bc46 100755 --- a/tools/launchdev.sh +++ b/tools/launchdev.sh @@ -304,7 +304,6 @@ function st2start(){ "${RUNNER_SCREENS[@]}" "st2-sensorcontainer" "st2-rulesengine" - "st2-resultstracker" "st2-notifier" "st2-auth" "st2-timersengine" From acb4af073d9c46387862f5d2e695793c698f3c37 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 24 Dec 2020 20:24:44 +0100 Subject: [PATCH 15/31] Remove the resultstracker from the tests st2.conf --- st2tests/conf/st2.conf | 3 --- 1 file changed, 3 deletions(-) diff --git a/st2tests/conf/st2.conf b/st2tests/conf/st2.conf index 9dadee986c..0801ccc4a7 100644 --- a/st2tests/conf/st2.conf +++ b/st2tests/conf/st2.conf @@ -50,6 +50,3 @@ remote_dir = /tmp [history] logging = st2actions/conf/logging.history.conf - -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf From 72cf5a88641fbd7fba7778a85e62f5f78c0259f3 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 24 Dec 2020 20:26:20 +0100 Subject: [PATCH 16/31] Remove overriden resultstracker config from the st2tests config.py --- st2tests/st2tests/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py index 77dca923ab..7fa4ad7b6e 100644 --- a/st2tests/st2tests/config.py +++ b/st2tests/st2tests/config.py @@ -89,7 +89,6 @@ def _override_common_opts(): CONF.set_override(name='packs_base_paths', override=packs_base_path, group='content') CONF.set_override(name='api_url', override='http://127.0.0.1', group='auth') CONF.set_override(name='mask_secrets', override=True, group='log') - CONF.set_override(name='query_interval', override=0.1, group='resultstracker') CONF.set_override(name='stream_output', override=False, group='actionrunner') From b2b31b2ffda937f403e970c3207b6082615a15ae Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 24 Dec 2020 20:28:48 +0100 Subject: [PATCH 17/31] Remove st2resultstracker from the st2test fixture configs --- .../st2tests/fixtures/conf/st2.tests.api.audit_log_level.conf | 4 ---- .../st2tests/fixtures/conf/st2.tests.api.debug_log_level.conf | 4 ---- .../st2tests/fixtures/conf/st2.tests.api.info_log_level.conf | 4 ---- .../fixtures/conf/st2.tests.api.system_debug_true.conf | 4 ---- .../conf/st2.tests.api.system_debug_true_logging_debug.conf | 4 ---- st2tests/st2tests/fixtures/conf/st2.tests.conf | 4 ---- 6 files changed, 24 deletions(-) diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.audit_log_level.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.audit_log_level.conf index feb8afd095..a07773bc48 100644 --- a/st2tests/st2tests/fixtures/conf/st2.tests.api.audit_log_level.conf +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.audit_log_level.conf @@ -85,10 +85,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.debug_log_level.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.debug_log_level.conf index 06e5eb0561..0b435df722 100644 --- a/st2tests/st2tests/fixtures/conf/st2.tests.api.debug_log_level.conf +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.debug_log_level.conf @@ -85,10 +85,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.info_log_level.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.info_log_level.conf index dd72dfd1b6..45bf5f364c 100644 --- a/st2tests/st2tests/fixtures/conf/st2.tests.api.info_log_level.conf +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.info_log_level.conf @@ -85,10 +85,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true.conf index 7fa1f26e40..ee8b2c9e07 100644 --- a/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true.conf +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true.conf @@ -85,10 +85,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true_logging_debug.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true_logging_debug.conf index 8172c6650a..caaf4bbfba 100644 --- a/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true_logging_debug.conf +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true_logging_debug.conf @@ -85,10 +85,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.conf b/st2tests/st2tests/fixtures/conf/st2.tests.conf index 694cfb082c..da7386382a 100644 --- a/st2tests/st2tests/fixtures/conf/st2.tests.conf +++ b/st2tests/st2tests/fixtures/conf/st2.tests.conf @@ -86,10 +86,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf From 4255459c1e44969ab404e98f69fef7a488eec3ed Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 24 Dec 2020 20:29:40 +0100 Subject: [PATCH 18/31] Remove the resultstracker from the HA sample config --- conf/HA/st2.conf.sample | 3 --- 1 file changed, 3 deletions(-) diff --git a/conf/HA/st2.conf.sample b/conf/HA/st2.conf.sample index 20cbd422ff..b52fe81bf4 100644 --- a/conf/HA/st2.conf.sample +++ b/conf/HA/st2.conf.sample @@ -23,9 +23,6 @@ logging = /etc/st2/logging.actionrunner.conf # The line should be commented and 'always-copy' removed when using EL7 or EL8 as it causes virtualenv issues on pack install virtualenv_opts = --always-copy -[resultstracker] -logging = /etc/st2/logging.resultstracker.conf - [notifier] logging = /etc/st2/logging.notifier.conf From 3f0aade8fc19be5e75a8ed6e10f7c6262727e317 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 24 Dec 2020 20:32:25 +0100 Subject: [PATCH 19/31] Remove the resultstracker test script from the st2action integration tests --- .../tests/integration/test_resultstracker.py | 378 ------------------ 1 file changed, 378 deletions(-) delete mode 100644 st2actions/tests/integration/test_resultstracker.py diff --git a/st2actions/tests/integration/test_resultstracker.py b/st2actions/tests/integration/test_resultstracker.py deleted file mode 100644 index d5b9a0422d..0000000000 --- a/st2actions/tests/integration/test_resultstracker.py +++ /dev/null @@ -1,378 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -import os -import imp - -import eventlet -import mock - -from oslo_config import cfg - -from st2common.constants import action as action_constants -from st2common.exceptions.db import StackStormDBObjectNotFoundError -from st2common.persistence.action import Action -from st2common.persistence.executionstate import ActionExecutionState -from st2common.persistence.liveaction import LiveAction -from st2common.runners import utils as runners_utils -from st2common.services import executions -from st2common.util import action_db as action_db_utils -from st2tests.base import (DbTestCase, EventletTestCase) -from st2tests.fixturesloader import FixturesLoader - -__all__ = [ - 'ResultsTrackerTests' -] - - -FIXTURES_LOADER = FixturesLoader() -FIXTURES_PACK = 'generic' -FIXTURES = { - 'actionstates': ['state1.yaml', 'state2.yaml'], - 'liveactions': ['liveaction1.yaml', 'liveaction2.yaml'] -} - -CURRENT_DIR = os.path.dirname(__file__) -ST2CONTENT_DIR = os.path.join(CURRENT_DIR, '../../../st2tests/st2tests/fixtures/packs/runners') - -MOCK_RUNNER_NAME = 'test_querymodule' - -MOCK_QUERIER_PATH = '{0}/{1}/query/{1}.py'.format(ST2CONTENT_DIR, MOCK_RUNNER_NAME) -MOCK_QUERIER_PATH = os.path.abspath(MOCK_QUERIER_PATH) -MOCK_QUERIER_MODULE = imp.load_source(MOCK_RUNNER_NAME + '.query', MOCK_QUERIER_PATH) - -MOCK_CALLBACK_PATH = '{0}/{1}/callback/{1}.py'.format(ST2CONTENT_DIR, MOCK_RUNNER_NAME) -MOCK_CALLBACK_PATH = os.path.abspath(MOCK_CALLBACK_PATH) -MOCK_CALLBACK_MODULE = imp.load_source(MOCK_RUNNER_NAME + '.callback', MOCK_CALLBACK_PATH) - - -@mock.patch.object( - executions, 'update_execution', mock.MagicMock(return_value=None)) -@mock.patch.object( - LiveAction, 'publish_update', mock.MagicMock(return_value=None)) -@mock.patch('st2common.runners.base.get_query_module', - mock.Mock(return_value=MOCK_QUERIER_MODULE)) -@mock.patch('st2actions.resultstracker.resultstracker.get_query_module', - mock.Mock(return_value=MOCK_QUERIER_MODULE)) -class ResultsTrackerTests(EventletTestCase, DbTestCase): - states = None - models = None - liveactions = None - - @classmethod - def setUpClass(cls): - super(ResultsTrackerTests, cls).setUpClass() - cfg.CONF.set_default('empty_q_sleep_time', 0.2, group='resultstracker') - cfg.CONF.set_default('no_workers_sleep_time', 0.1, group='resultstracker') - - @classmethod - def tearDownClass(cls): - cfg.CONF.set_default('empty_q_sleep_time', 1, group='resultstracker') - cfg.CONF.set_default('no_workers_sleep_time', 1, group='resultstracker') - super(ResultsTrackerTests, cls).tearDownClass() - - def setUp(self): - super(ResultsTrackerTests, self).setUp() - DbTestCase.setUpClass() - ResultsTrackerTests.models = FIXTURES_LOADER.save_fixtures_to_db( - fixtures_pack=FIXTURES_PACK, fixtures_dict=FIXTURES) - ResultsTrackerTests.states = ResultsTrackerTests.models['actionstates'] - ResultsTrackerTests.liveactions = ResultsTrackerTests.models['liveactions'] - ResultsTrackerTests._update_state_models() - - def tearDown(self): - FIXTURES_LOADER.delete_models_from_db(ResultsTrackerTests.models) - super(ResultsTrackerTests, self).tearDown() - - @mock.patch.object( - Action, 'get_by_ref', mock.MagicMock(return_value='foobar')) - def test_query_process(self): - tracker = self._get_tracker() - runners_utils.invoke_post_run = mock.Mock(return_value=None) - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - # Process the state objects. - tracker._bootstrap() - eventlet.sleep(1) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertIsNotNone(exec_db.result['called_with'][exec_id], exec_db.result) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertIsNotNone(exec_db.result['called_with'][exec_id], exec_db.result) - - tracker.shutdown() - - # Ensure state objects are deleted. - self.assertRaises( - StackStormDBObjectNotFoundError, - ActionExecutionState.get_by_id, - ResultsTrackerTests.states['state1.yaml'].id - ) - - self.assertRaises( - StackStormDBObjectNotFoundError, - ActionExecutionState.get_by_id, - ResultsTrackerTests.states['state2.yaml'].id - ) - - # Ensure invoke_post_run is called. - self.assertEqual(2, runners_utils.invoke_post_run.call_count) - - def test_start_shutdown(self): - tracker = self._get_tracker() - tracker.start() - eventlet.sleep(0.1) - tracker.shutdown() - - def test_get_querier_success(self): - tracker = self._get_tracker() - self.assertIsNotNone(tracker.get_querier('test_querymodule')) - - def test_get_querier_not_found(self): - with mock.patch('st2actions.resultstracker.resultstracker.get_query_module', - mock.Mock(side_effect=Exception('Not found'))): - tracker = self._get_tracker() - self.assertEqual(tracker.get_querier('this_module_aint_exist'), None) - - def test_querier_started(self): - tracker = self._get_tracker() - querier = tracker.get_querier('test_querymodule') - eventlet.sleep(0.1) - self.assertTrue(querier.is_started(), 'querier must have been started.') - - def test_delete_state_object_on_error_at_query(self): - tracker = self._get_tracker() - querier = tracker.get_querier('test_querymodule') - querier._delete_state_object = mock.Mock(return_value=None) - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - with mock.patch.object( - querier.__class__, 'query', - mock.MagicMock(side_effect=Exception('Mock query exception.'))): - tracker._bootstrap() - eventlet.sleep(1) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - tracker.shutdown() - - # Ensure deletes are called. - self.assertEqual(2, querier._delete_state_object.call_count) - - def test_keep_state_object_on_error_at_query(self): - tracker = self._get_tracker() - querier = tracker.get_querier('test_querymodule') - querier._delete_state_object = mock.Mock(return_value=None) - querier.delete_state_object_on_error = False - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - with mock.patch.object( - querier.__class__, 'query', - mock.MagicMock(side_effect=Exception('Mock query exception.'))): - tracker._bootstrap() - eventlet.sleep(1) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - tracker.shutdown() - - # Ensure deletes are not called. - querier._delete_state_object.assert_not_called() - - def test_delete_state_object_on_error_at_update_result(self): - tracker = self._get_tracker() - querier = tracker.get_querier('test_querymodule') - querier._delete_state_object = mock.Mock(return_value=None) - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - with mock.patch.object( - querier.__class__, '_update_action_results', - mock.MagicMock(side_effect=Exception('Mock update exception.'))): - tracker._bootstrap() - eventlet.sleep(1) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - tracker.shutdown() - - # Ensure deletes are called. - self.assertEqual(2, querier._delete_state_object.call_count) - - def test_keep_state_object_on_error_at_update_result(self): - tracker = self._get_tracker() - querier = tracker.get_querier('test_querymodule') - querier._delete_state_object = mock.Mock(return_value=None) - querier.delete_state_object_on_error = False - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - with mock.patch.object( - querier.__class__, '_update_action_results', - mock.MagicMock(side_effect=Exception('Mock update exception.'))): - tracker._bootstrap() - eventlet.sleep(1) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - tracker.shutdown() - - # Ensure deletes are not called. - querier._delete_state_object.assert_not_called() - - @mock.patch.object( - Action, 'get_by_ref', mock.MagicMock(return_value='foobar')) - def test_execution_cancellation(self): - tracker = self._get_tracker() - querier = tracker.get_querier('test_querymodule') - querier._delete_state_object = mock.Mock(return_value=None) - runners_utils.invoke_post_run = mock.Mock(return_value=None) - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - with mock.patch.object( - querier.__class__, 'query', - mock.MagicMock(return_value=(action_constants.LIVEACTION_STATUS_CANCELED, {}))): - tracker._bootstrap() - eventlet.sleep(2) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - tracker.shutdown() - - # Ensure deletes are called. - self.assertEqual(2, querier._delete_state_object.call_count) - - # Ensure invoke_post_run is called. - self.assertEqual(2, runners_utils.invoke_post_run.call_count) - - @mock.patch.object( - Action, 'get_by_ref', mock.MagicMock(return_value=None)) - def test_action_deleted(self): - tracker = self._get_tracker() - action_db_utils.get_runnertype_by_name = mock.Mock(return_value=None) - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - # Process the state objects. - tracker._bootstrap() - eventlet.sleep(1) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertIsNotNone(exec_db.result['called_with'][exec_id], exec_db.result) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertIsNotNone(exec_db.result['called_with'][exec_id], exec_db.result) - - tracker.shutdown() - - # Ensure state objects are deleted. - self.assertRaises( - StackStormDBObjectNotFoundError, - ActionExecutionState.get_by_id, - ResultsTrackerTests.states['state1.yaml'].id - ) - - self.assertRaises( - StackStormDBObjectNotFoundError, - ActionExecutionState.get_by_id, - ResultsTrackerTests.states['state2.yaml'].id - ) - - # Ensure get_runnertype_by_name in invoke_post_run is not called. - action_db_utils.get_runnertype_by_name.assert_not_called() - - def _get_tracker(self): - from st2actions.resultstracker import resultstracker - tracker = resultstracker.get_tracker() - return tracker - - @classmethod - def _update_state_models(cls): - states = ResultsTrackerTests.states - state1 = ActionExecutionState.get_by_id(states['state1.yaml'].id) - state1.execution_id = ResultsTrackerTests.liveactions['liveaction1.yaml'].id - state2 = ActionExecutionState.get_by_id(states['state2.yaml'].id) - state2.execution_id = ResultsTrackerTests.liveactions['liveaction2.yaml'].id - ResultsTrackerTests.states['state1.yaml'] = ActionExecutionState.add_or_update(state1) - ResultsTrackerTests.states['state2.yaml'] = ActionExecutionState.add_or_update(state2) From aed61f1e2e27e41f588003315a1cde7c28f4aa82 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Sat, 16 Jan 2021 22:29:14 +0100 Subject: [PATCH 20/31] Reve st2resultstracker code from st2actions --- .../st2actions/cmd/st2resultstracker.py | 74 ----------- .../st2actions/resultstracker/__init__.py | 0 .../st2actions/resultstracker/config.py | 57 --------- .../resultstracker/resultstracker.py | 115 ------------------ .../integration/test_action_state_consumer.py | 81 ------------ st2common/st2common/query/base.py | 3 - 6 files changed, 330 deletions(-) delete mode 100644 st2actions/st2actions/cmd/st2resultstracker.py delete mode 100644 st2actions/st2actions/resultstracker/__init__.py delete mode 100644 st2actions/st2actions/resultstracker/config.py delete mode 100644 st2actions/st2actions/resultstracker/resultstracker.py delete mode 100644 st2actions/tests/integration/test_action_state_consumer.py diff --git a/st2actions/st2actions/cmd/st2resultstracker.py b/st2actions/st2actions/cmd/st2resultstracker.py deleted file mode 100644 index 69d4348055..0000000000 --- a/st2actions/st2actions/cmd/st2resultstracker.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -from st2common.util.monkey_patch import monkey_patch -monkey_patch() - -import os -import sys - -from st2common import log as logging -from st2common.service_setup import setup as common_setup -from st2common.service_setup import teardown as common_teardown -from st2actions.resultstracker import config -from st2actions.resultstracker import resultstracker - -__all__ = [ - 'main' -] - - -LOG = logging.getLogger(__name__) - - -def _setup(): - capabilities = { - 'name': 'resultstracker', - 'type': 'passive' - } - common_setup(service='resultstracker', config=config, setup_db=True, register_mq_exchanges=True, - register_signal_handlers=True, service_registry=True, capabilities=capabilities) - - -def _run_worker(): - LOG.info('(PID=%s) Results tracker started.', os.getpid()) - tracker = resultstracker.get_tracker() - try: - tracker.start(wait=True) - except (KeyboardInterrupt, SystemExit): - LOG.info('(PID=%s) Results tracker stopped.', os.getpid()) - tracker.shutdown() - except: - return 1 - return 0 - - -def _teardown(): - common_teardown() - - -def main(): - try: - _setup() - return _run_worker() - except SystemExit as exit_code: - sys.exit(exit_code) - except: - LOG.exception('(PID=%s) Results tracker quit due to exception.', os.getpid()) - return 1 - finally: - _teardown() diff --git a/st2actions/st2actions/resultstracker/__init__.py b/st2actions/st2actions/resultstracker/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2actions/st2actions/resultstracker/config.py b/st2actions/st2actions/resultstracker/config.py deleted file mode 100644 index 0e48c48dd4..0000000000 --- a/st2actions/st2actions/resultstracker/config.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -from oslo_config import cfg - -from st2common import config as common_config -from st2common.constants.system import VERSION_STRING -from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH - -common_config.register_opts() - -CONF = cfg.CONF - - -def parse_args(args=None): - cfg.CONF(args=args, version=VERSION_STRING, - default_config_files=[DEFAULT_CONFIG_FILE_PATH]) - - -def register_opts(): - _register_common_opts() - _register_results_tracker_opts() - - -def get_logging_config_path(): - return cfg.CONF.resultstracker.logging - - -def _register_common_opts(): - common_config.register_opts() - - -def _register_results_tracker_opts(): - resultstracker_opts = [ - cfg.StrOpt( - 'logging', default='/etc/st2/logging.resultstracker.conf', - help='Location of the logging configuration file.') - ] - - CONF.register_opts(resultstracker_opts, group='resultstracker') - - -register_opts() diff --git a/st2actions/st2actions/resultstracker/resultstracker.py b/st2actions/st2actions/resultstracker/resultstracker.py deleted file mode 100644 index dd8b618181..0000000000 --- a/st2actions/st2actions/resultstracker/resultstracker.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -import eventlet -import six - -from collections import defaultdict - -from st2common.query.base import QueryContext -from st2common import log as logging -from st2common.models.db.executionstate import ActionExecutionStateDB -from st2common.persistence.executionstate import ActionExecutionState -from st2common.transport import consumers -from st2common.transport import utils as transport_utils -from st2common.runners.base import get_query_module -from st2common.transport.queues import RESULTSTRACKER_ACTIONSTATE_WORK_QUEUE - -__all__ = [ - 'ResultsTracker', - 'get_tracker' -] - - -LOG = logging.getLogger(__name__) - - -class ResultsTracker(consumers.MessageHandler): - message_type = ActionExecutionStateDB - - def __init__(self, connection, queues): - super(ResultsTracker, self).__init__(connection, queues) - self._queriers = {} - self._query_threads = [] - self._failed_imports = set() - - def start(self, wait=False): - self._bootstrap() - super(ResultsTracker, self).start(wait=wait) - - def wait(self): - super(ResultsTracker, self).wait() - for thread in self._query_threads: - thread.wait() - - def shutdown(self): - super(ResultsTracker, self).shutdown() - LOG.info('Stats from queriers:') - self._print_stats() - - def _print_stats(self): - for _, querier in six.iteritems(self._queriers): - if querier: - querier.print_stats() - - def _bootstrap(self): - all_states = ActionExecutionState.get_all() - LOG.info('Found %d pending states in db.' % len(all_states)) - - query_contexts_dict = defaultdict(list) - for state_db in all_states: - try: - context = QueryContext.from_model(state_db) - except: - LOG.exception('Invalid state object: %s', state_db) - continue - query_module_name = state_db.query_module - querier = self.get_querier(query_module_name) - - if querier is not None: - query_contexts_dict[querier].append(context) - - for querier, contexts in six.iteritems(query_contexts_dict): - LOG.info('Found %d pending actions for query module %s', len(contexts), querier) - querier.add_queries(query_contexts=contexts) - - def process(self, query_context): - querier = self.get_querier(query_context.query_module) - context = QueryContext.from_model(query_context) - querier.add_queries(query_contexts=[context]) - return - - def get_querier(self, query_module_name): - if (query_module_name not in self._queriers and - query_module_name not in self._failed_imports): - try: - query_module = get_query_module(query_module_name) - except: - LOG.exception('Failed importing query module: %s', query_module_name) - self._failed_imports.add(query_module_name) - self._queriers[query_module_name] = None - else: - querier = query_module.get_instance() - self._queriers[query_module_name] = querier - self._query_threads.append(eventlet.spawn(querier.start)) - - return self._queriers[query_module_name] - - -def get_tracker(): - with transport_utils.get_connection() as conn: - return ResultsTracker(conn, [RESULTSTRACKER_ACTIONSTATE_WORK_QUEUE]) diff --git a/st2actions/tests/integration/test_action_state_consumer.py b/st2actions/tests/integration/test_action_state_consumer.py deleted file mode 100644 index 4cb0d05db0..0000000000 --- a/st2actions/tests/integration/test_action_state_consumer.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -import os -import imp - -import mock - -from st2common.transport.queues import RESULTSTRACKER_ACTIONSTATE_WORK_QUEUE -from st2actions.resultstracker.resultstracker import ResultsTracker -from st2common.models.db.executionstate import ActionExecutionStateDB -from st2common.persistence.executionstate import ActionExecutionState -from st2common.transport import utils as transport_utils -from st2tests.base import DbTestCase, EventletTestCase -from st2tests.fixturesloader import FixturesLoader -from st2tests.fixtures.packs.runners.test_querymodule.query.test_querymodule import TestQuerier - -FIXTURES_PACK = 'generic' -FIXTURES = {'liveactions': ['liveaction1.yaml']} -loader = FixturesLoader() - -CURRENT_DIR = os.path.dirname(__file__) -ST2CONTENT_DIR = os.path.join(CURRENT_DIR, '../../../st2tests/st2tests/fixtures/packs/runners') - -MOCK_RUNNER_NAME = 'test_querymodule' - -MOCK_QUERIER_PATH = '{0}/{1}/query/{1}.py'.format(ST2CONTENT_DIR, MOCK_RUNNER_NAME) -MOCK_QUERIER_PATH = os.path.abspath(MOCK_QUERIER_PATH) -MOCK_QUERIER_MODULE = imp.load_source(MOCK_RUNNER_NAME + '.query', MOCK_QUERIER_PATH) - - -@mock.patch('st2common.runners.base.get_query_module', - mock.Mock(return_value=MOCK_QUERIER_MODULE)) -@mock.patch('st2actions.resultstracker.resultstracker.get_query_module', - mock.Mock(return_value=MOCK_QUERIER_MODULE)) -class ActionStateConsumerTests(EventletTestCase, DbTestCase): - models = None - liveactions = None - - @classmethod - def setUpClass(cls): - super(ActionStateConsumerTests, cls).setUpClass() - DbTestCase.setUpClass() - cls.models = loader.save_fixtures_to_db(fixtures_pack=FIXTURES_PACK, - fixtures_dict=FIXTURES) - cls.liveactions = cls.models['liveactions'] - - @mock.patch.object(TestQuerier, 'query', mock.MagicMock(return_value=(False, {}))) - def test_process_message(self): - with transport_utils.get_connection() as conn: - tracker = ResultsTracker(conn, [RESULTSTRACKER_ACTIONSTATE_WORK_QUEUE]) - tracker._bootstrap() - state = ActionStateConsumerTests.get_state( - ActionStateConsumerTests.liveactions['liveaction1.yaml']) - tracker._queue_consumer._process_message(state) - querier = tracker.get_querier('test_querymodule') - self.assertEqual(querier._query_contexts.qsize(), 1) - - @classmethod - def get_state(cls, exec_db): - state = ActionExecutionStateDB(execution_id=str(exec_db.id), query_context={'id': 'foo'}, - query_module='test_querymodule') - return ActionExecutionState.add_or_update(state) - - @classmethod - def tearDownClass(cls): - loader.delete_models_from_db(ActionStateConsumerTests.models) diff --git a/st2common/st2common/query/base.py b/st2common/st2common/query/base.py index 83c03e55d0..7267439c62 100644 --- a/st2common/st2common/query/base.py +++ b/st2common/st2common/query/base.py @@ -51,9 +51,6 @@ def _get_config_value(self, config_option): LOG.warning('You are using deprecated config group "results_tracker" for "%s". ' 'Please use "resultstracker" group instead.', config_option) - if not config_value and config_option in cfg.CONF.resultstracker: - config_value = getattr(cfg.CONF.resultstracker, config_option) - return config_value def __init__(self): From a904de54c5135232aab331bc1f31eead422c0080 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Sun, 17 Jan 2021 21:31:57 +0100 Subject: [PATCH 21/31] Delete the st2common query module along with related tests --- st2common/st2common/query/__init__.py | 0 st2common/st2common/query/base.py | 240 ------------------ .../runners/mock_query_callback/MANIFEST.in | 11 - .../mock_query_callback/callback/__init__.py | 0 .../runners/mock_query_callback/dist_utils.py | 169 ------------ .../mock_query_callback/in-requirements.txt | 0 .../mock_query_callback/__init__.py | 16 -- .../mock_query_callback/callback.py | 29 --- .../mock_query_callback.py | 60 ----- .../mock_query_callback/query.py | 33 --- .../mock_query_callback/runner.yaml | 7 - .../mock_query_callback/query/__init__.py | 0 .../mock_query_callback/requirements.txt | 8 - .../runners/mock_query_callback/runner.yaml | 1 - .../runners/mock_query_callback/setup.py | 61 ----- st2common/tests/unit/test_query_base.py | 219 ---------------- .../runners/test_querymodule/__init__.py | 0 .../test_querymodule/callback/__init__.py | 0 .../callback/test_querymodule.py | 32 --- .../test_querymodule/query/__init__.py | 0 .../query/test_querymodule.py | 30 --- 21 files changed, 916 deletions(-) delete mode 100644 st2common/st2common/query/__init__.py delete mode 100644 st2common/st2common/query/base.py delete mode 100644 st2common/tests/runners/mock_query_callback/MANIFEST.in delete mode 100644 st2common/tests/runners/mock_query_callback/callback/__init__.py delete mode 100644 st2common/tests/runners/mock_query_callback/dist_utils.py delete mode 100644 st2common/tests/runners/mock_query_callback/in-requirements.txt delete mode 100644 st2common/tests/runners/mock_query_callback/mock_query_callback/__init__.py delete mode 100644 st2common/tests/runners/mock_query_callback/mock_query_callback/callback.py delete mode 100644 st2common/tests/runners/mock_query_callback/mock_query_callback/mock_query_callback.py delete mode 100644 st2common/tests/runners/mock_query_callback/mock_query_callback/query.py delete mode 100644 st2common/tests/runners/mock_query_callback/mock_query_callback/runner.yaml delete mode 100644 st2common/tests/runners/mock_query_callback/query/__init__.py delete mode 100644 st2common/tests/runners/mock_query_callback/requirements.txt delete mode 120000 st2common/tests/runners/mock_query_callback/runner.yaml delete mode 100644 st2common/tests/runners/mock_query_callback/setup.py delete mode 100644 st2common/tests/unit/test_query_base.py delete mode 100644 st2tests/st2tests/fixtures/packs/runners/test_querymodule/__init__.py delete mode 100644 st2tests/st2tests/fixtures/packs/runners/test_querymodule/callback/__init__.py delete mode 100644 st2tests/st2tests/fixtures/packs/runners/test_querymodule/callback/test_querymodule.py delete mode 100644 st2tests/st2tests/fixtures/packs/runners/test_querymodule/query/__init__.py delete mode 100644 st2tests/st2tests/fixtures/packs/runners/test_querymodule/query/test_querymodule.py diff --git a/st2common/st2common/query/__init__.py b/st2common/st2common/query/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2common/st2common/query/base.py b/st2common/st2common/query/base.py deleted file mode 100644 index 7267439c62..0000000000 --- a/st2common/st2common/query/base.py +++ /dev/null @@ -1,240 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -import abc -import eventlet -import six.moves.queue -import six -import time - -from oslo_config import cfg - -from st2common import log as logging -from st2common.constants import action as action_constants -from st2common.exceptions import db as db_exc -from st2common.persistence.executionstate import ActionExecutionState -from st2common.persistence.liveaction import LiveAction -from st2common.runners import utils as runners_utils -from st2common.services import executions -from st2common.util import date as date_utils - -LOG = logging.getLogger(__name__) - -__all__ = [ - 'Querier', - 'QueryContext' -] - - -@six.add_metaclass(abc.ABCMeta) -class Querier(object): - delete_state_object_on_error = True - - def _get_config_value(self, config_option): - config_value = None - - if 'results_tracker' in cfg.CONF and config_option in cfg.CONF.results_tracker: - config_value = getattr(cfg.CONF.results_tracker, config_option) - LOG.warning('You are using deprecated config group "results_tracker" for "%s". ' - 'Please use "resultstracker" group instead.', config_option) - - return config_value - - def __init__(self): - self._empty_q_sleep_time = self._get_config_value('empty_q_sleep_time') - self._no_workers_sleep_time = self._get_config_value('no_workers_sleep_time') - self._query_interval = self._get_config_value('query_interval') - self._query_thread_pool_size = self._get_config_value('thread_pool_size') - self._query_contexts = six.moves.queue.Queue() - self._thread_pool = eventlet.GreenPool(self._query_thread_pool_size) - self._started = False - - def start(self): - self._started = True - while True: - while self._query_contexts.empty(): - eventlet.greenthread.sleep(self._empty_q_sleep_time) - while self._thread_pool.free() <= 0: - eventlet.greenthread.sleep(self._no_workers_sleep_time) - self._fire_queries() - eventlet.sleep(self._query_interval) - - def add_queries(self, query_contexts=None): - if query_contexts is None: - query_contexts = [] - LOG.debug('Adding queries to querier: %s' % query_contexts) - for query_context in query_contexts: - self._query_contexts.put((time.time(), query_context)) - - def is_started(self): - return self._started - - def _fire_queries(self, blocking=False): - if self._thread_pool.free() <= 0: - return - - now = time.time() - reschedule_queries = [] - - while not self._query_contexts.empty() and self._thread_pool.free() > 0: - (last_query_time, query_context) = self._query_contexts.get_nowait() - if now - last_query_time < self._query_interval: - reschedule_queries.append((last_query_time, query_context)) - continue - else: - if not blocking: - self._thread_pool.spawn( - self._query_and_save_results, - query_context, - last_query_time - ) - # Add an option to block and execute the function directly for unit tests. - else: - self._query_and_save_results( - query_context, - last_query_time - ) - - for query in reschedule_queries: - self._query_contexts.put((query[0], query[1])) - - def _query_and_save_results(self, query_context, last_query_time=None): - this_query_time = time.time() - execution_id = query_context.execution_id - actual_query_context = query_context.query_context - - LOG.debug('Querying external service for results. Context: %s' % actual_query_context) - try: - (status, results) = self.query( - execution_id, - actual_query_context, - last_query_time=last_query_time - ) - except: - LOG.exception('Failed querying results for liveaction_id %s.', execution_id) - if self.delete_state_object_on_error: - self._delete_state_object(query_context) - LOG.debug('Removed state object %s.', query_context) - return - - liveaction_db = None - try: - liveaction_db = self._update_action_results(execution_id, status, results) - except Exception: - LOG.exception('Failed updating action results for liveaction_id %s', execution_id) - if self.delete_state_object_on_error: - self._delete_state_object(query_context) - LOG.debug('Removed state object %s.', query_context) - return - - if (status in action_constants.LIVEACTION_COMPLETED_STATES or - status == action_constants.LIVEACTION_STATUS_PAUSED): - runners_utils.invoke_post_run(liveaction_db) - self._delete_state_object(query_context) - LOG.debug( - "Detailed workflow liveaction results - ", extra={'liveaction_db': liveaction_db} - ) - return - - if not self._is_state_object_exist(query_context): - LOG.warning( - 'Query for liveaction_id %s is not rescheduled ' - 'because state object %s has been deleted.', - execution_id, - query_context.id - ) - - return - - self._query_contexts.put((this_query_time, query_context)) - - def _update_action_results(self, execution_id, status, results): - liveaction_db = LiveAction.get_by_id(execution_id) - if not liveaction_db: - raise Exception('No DB model for liveaction_id: %s' % execution_id) - - if liveaction_db.status != action_constants.LIVEACTION_STATUS_CANCELED: - liveaction_db.status = status - - liveaction_db.result = results - - # Action has completed, record end_timestamp - if (liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES and - not liveaction_db.end_timestamp): - liveaction_db.end_timestamp = date_utils.get_datetime_utc_now() - - # update liveaction, update actionexecution and then publish update. - updated_liveaction = LiveAction.add_or_update(liveaction_db, publish=False) - executions.update_execution(updated_liveaction) - LiveAction.publish_update(updated_liveaction) - - return updated_liveaction - - def _delete_state_object(self, query_context): - state_db = None - - try: - state_db = ActionExecutionState.get_by_id(query_context.id) - except db_exc.StackStormDBObjectNotFoundError: - pass - - if state_db is not None: - try: - LOG.info('Clearing state object: %s', state_db) - ActionExecutionState.delete(state_db) - except: - LOG.exception('Failed clearing state object: %s', state_db) - - def _is_state_object_exist(self, query_context): - state_db = None - - try: - state_db = ActionExecutionState.get_by_id(query_context.id) - except db_exc.StackStormDBObjectNotFoundError: - pass - - return (state_db is not None) - - def query(self, execution_id, query_context, last_query_time=None): - """ - This is the method individual queriers must implement. - This method should return a tuple of (status, results). - - status should be one of LIVEACTION_STATUS_SUCCEEDED, LIVEACTION_STATUS_RUNNING, - LIVEACTION_STATUS_FAILED defined in st2common.constants.action. - """ - raise NotImplementedError() - - def print_stats(self): - LOG.info('\t --- Name: %s, pending queuries: %d', self.__class__.__name__, - self._query_contexts.qsize()) - - -class QueryContext(object): - def __init__(self, obj_id, execution_id, query_context, query_module): - self.id = obj_id - self.execution_id = execution_id - self.query_context = query_context - self.query_module = query_module - - @classmethod - def from_model(cls, model): - return QueryContext(str(model.id), str(model.execution_id), model.query_context, - model.query_module) - - def __repr__(self): - return ('' % - (self.id, self.execution_id, self.query_context)) diff --git a/st2common/tests/runners/mock_query_callback/MANIFEST.in b/st2common/tests/runners/mock_query_callback/MANIFEST.in deleted file mode 100644 index 25ce80c091..0000000000 --- a/st2common/tests/runners/mock_query_callback/MANIFEST.in +++ /dev/null @@ -1,11 +0,0 @@ -# https://docs.python.org/2/distutils/sourcedist.html#commands -# Include all files under the source tree by default. -# Another behaviour can be used in the future though. -include __init__.py -include runner.yaml -include dist_utils.py -include requirements.txt -include README.rst -include CHANGELOG.rst -include LICENSE -global-exclude *.pyc diff --git a/st2common/tests/runners/mock_query_callback/callback/__init__.py b/st2common/tests/runners/mock_query_callback/callback/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2common/tests/runners/mock_query_callback/dist_utils.py b/st2common/tests/runners/mock_query_callback/dist_utils.py deleted file mode 100644 index 1d637ca664..0000000000 --- a/st2common/tests/runners/mock_query_callback/dist_utils.py +++ /dev/null @@ -1,169 +0,0 @@ -# -*- coding: utf-8 -*- -# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY -# Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to -# update dist_utils.py files for all components - -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -import os -import re -import sys - -from distutils.version import StrictVersion - -# NOTE: This script can't rely on any 3rd party dependency so we need to use this code here -# -# TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import -# from pip? -# -# TODO: Dear future developer, if you are back here fixing a bug with how we parse -# requirements files, please look into using the packaging package on PyPI: -# https://packaging.pypa.io/en/latest/requirements/ -# and specifying that in the `setup_requires` argument to `setuptools.setup()` -# for subpackages. -# At the very least we can vendorize some of their code instead of reimplementing -# each piece of their code every time our parsing breaks. -PY3 = sys.version_info[0] == 3 - -if PY3: - text_type = str -else: - text_type = unicode # NOQA - -GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' - -__all__ = [ - 'check_pip_is_installed', - 'check_pip_version', - 'fetch_requirements', - 'apply_vagrant_workaround', - 'get_version_string', - 'parse_version_string' -] - - -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print('Failed to import pip: %s' % (text_type(e))) - print('') - print('Download pip:\n%s' % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version='6.0.0'): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print("Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, - min_version, - GET_PIP)) - sys.exit(1) - - return True - - -def fetch_requirements(requirements_file_path): - """ - Return a list of requirements and links by parsing the provided requirements file. - """ - links = [] - reqs = [] - - def _get_link(line): - vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] - - for vcs_prefix in vcs_prefixes: - if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): - req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) - - if not req_name: - req_name = re.findall('.*#egg=(.+?)$', line) - else: - req_name = req_name[0] - - if not req_name: - raise ValueError('Line "%s" is missing "#egg="' % (line)) - - link = line.replace('-e ', '').strip() - return link, req_name[0] - - return None, None - - with open(requirements_file_path, 'r') as fp: - for line in fp.readlines(): - line = line.strip() - - if line.startswith('#') or not line: - continue - - link, req_name = _get_link(line=line) - - if link: - links.append(link) - else: - req_name = line - - if ';' in req_name: - req_name = req_name.split(';')[0].strip() - - reqs.append(req_name) - - return (reqs, links) - - -def apply_vagrant_workaround(): - """ - Function which detects if the script is being executed inside vagrant and if it is, it deletes - "os.link" attribute. - Note: Without this workaround, setup.py sdist will fail when running inside a shared directory - (nfs / virtualbox shared folders). - """ - if os.environ.get('USER', None) == 'vagrant': - del os.link - - -def get_version_string(init_file): - """ - Read __version__ string for an init file. - """ - - with open(init_file, 'r') as fp: - content = fp.read() - version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", - content, re.M) - if version_match: - return version_match.group(1) - - raise RuntimeError('Unable to find version string in %s.' % (init_file)) - - -# alias for get_version_string -parse_version_string = get_version_string diff --git a/st2common/tests/runners/mock_query_callback/in-requirements.txt b/st2common/tests/runners/mock_query_callback/in-requirements.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2common/tests/runners/mock_query_callback/mock_query_callback/__init__.py b/st2common/tests/runners/mock_query_callback/mock_query_callback/__init__.py deleted file mode 100644 index e682f5e7b6..0000000000 --- a/st2common/tests/runners/mock_query_callback/mock_query_callback/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__version__ = '3.3dev' diff --git a/st2common/tests/runners/mock_query_callback/mock_query_callback/callback.py b/st2common/tests/runners/mock_query_callback/mock_query_callback/callback.py deleted file mode 100644 index 9f41758bab..0000000000 --- a/st2common/tests/runners/mock_query_callback/mock_query_callback/callback.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -from st2common.callback import base as callback - - -def get_instance(): - return MockCallbackHandler - - -class MockCallbackHandler(callback.AsyncActionExecutionCallbackHandler): - - @classmethod - def callback(cls, liveaction): - pass diff --git a/st2common/tests/runners/mock_query_callback/mock_query_callback/mock_query_callback.py b/st2common/tests/runners/mock_query_callback/mock_query_callback/mock_query_callback.py deleted file mode 100644 index 7c78fcc134..0000000000 --- a/st2common/tests/runners/mock_query_callback/mock_query_callback/mock_query_callback.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED -from st2common.runners.base import ActionRunner -from st2common.runners.base import get_metadata as get_runner_metadata - -import uuid - - -__all__ = [ - 'MockQueryCallbackRunner', - - 'get_runner', - 'get_metadata' -] - - -class MockQueryCallbackRunner(ActionRunner): - """ - Runner which does absolutely nothing. No-op action. - """ - - def __init__(self, runner_id): - super(MockQueryCallbackRunner, self).__init__(runner_id=runner_id) - - def pre_run(self): - super(MockQueryCallbackRunner, self).pre_run() - - def run(self, action_parameters): - result = { - 'failed': False, - 'succeeded': True, - 'return_code': 0, - } - - status = LIVEACTION_STATUS_SUCCEEDED - return (status, result, None) - - -def get_runner(): - return MockQueryCallbackRunner(str(uuid.uuid4())) - - -def get_metadata(): - return get_runner_metadata('noop_runner')[0] diff --git a/st2common/tests/runners/mock_query_callback/mock_query_callback/query.py b/st2common/tests/runners/mock_query_callback/mock_query_callback/query.py deleted file mode 100644 index 065dd5d0e6..0000000000 --- a/st2common/tests/runners/mock_query_callback/mock_query_callback/query.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -from st2common.query.base import Querier -from st2common.constants import action as action_constants - - -def get_instance(): - return MockQueryCallbackQuerier() - - -class MockQueryCallbackQuerier(Querier): - - def __init__(self, *args, **kwargs): - super(MockQueryCallbackQuerier, self).__init__(*args, **kwargs) - - def query(self, execution_id, query_context, last_query_time=None): - return (action_constants.LIVEACTION_STATUS_SUCCEEDED, - {'called_with': {execution_id: query_context}}) diff --git a/st2common/tests/runners/mock_query_callback/mock_query_callback/runner.yaml b/st2common/tests/runners/mock_query_callback/mock_query_callback/runner.yaml deleted file mode 100644 index 318f0f555b..0000000000 --- a/st2common/tests/runners/mock_query_callback/mock_query_callback/runner.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- aliases: [] - description: A mock runner for query and callback - enabled: true - name: mock_query_callback - query_module: mock_query_callback - runner_module: mock_query_callback - diff --git a/st2common/tests/runners/mock_query_callback/query/__init__.py b/st2common/tests/runners/mock_query_callback/query/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2common/tests/runners/mock_query_callback/requirements.txt b/st2common/tests/runners/mock_query_callback/requirements.txt deleted file mode 100644 index f4a1c5a5ee..0000000000 --- a/st2common/tests/runners/mock_query_callback/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Don't edit this file. It's generated automatically! -# If you want to update global dependencies, modify fixed-requirements.txt -# and then run 'make requirements' to update requirements.txt for all -# components. -# If you want to update depdencies for a single component, modify the -# in-requirements.txt for that component and then run 'make requirements' to -# update the component requirements.txt - diff --git a/st2common/tests/runners/mock_query_callback/runner.yaml b/st2common/tests/runners/mock_query_callback/runner.yaml deleted file mode 120000 index 994fdf0c6b..0000000000 --- a/st2common/tests/runners/mock_query_callback/runner.yaml +++ /dev/null @@ -1 +0,0 @@ -mock_query_callback/runner.yaml \ No newline at end of file diff --git a/st2common/tests/runners/mock_query_callback/setup.py b/st2common/tests/runners/mock_query_callback/setup.py deleted file mode 100644 index af00f5b9cc..0000000000 --- a/st2common/tests/runners/mock_query_callback/setup.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -import os.path - -from setuptools import setup -from setuptools import find_packages - -from dist_utils import fetch_requirements -from dist_utils import apply_vagrant_workaround - -from mock_query_callback import __version__ - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -REQUIREMENTS_FILE = os.path.join(BASE_DIR, 'requirements.txt') - -install_reqs, dep_links = fetch_requirements(REQUIREMENTS_FILE) - -apply_vagrant_workaround() -setup( - name='stackstorm-runner-mock_query_callback', - version=__version__, - description=('Mock runner for query callback'), - author='StackStorm', - author_email='info@stackstorm.com', - license='Apache License (2.0)', - url='https://stackstorm.com/', - install_requires=install_reqs, - dependency_links=dep_links, - test_suite='tests', - zip_safe=False, - include_package_data=True, - packages=find_packages(exclude=['setuptools', 'tests']), - package_data={'mock_query_callback': ['runner.yaml']}, - scripts=[], - entry_points={ - 'st2common.runners.runner': [ - 'mock_query_callback = mock_query_callback.mock_query_callback', - ], - 'st2common.runners.query': [ - 'mock_query_callback = mock_query_callback.query', - ], - 'st2common.runners.callback': [ - 'mock_query_callback = mock_query_callback.callback', - ], - } -) diff --git a/st2common/tests/unit/test_query_base.py b/st2common/tests/unit/test_query_base.py deleted file mode 100644 index a1edf0610a..0000000000 --- a/st2common/tests/unit/test_query_base.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -import six.moves.queue -import time -import uuid - -from unittest2 import TestCase -import mock - -from st2common.constants import action as action_constants -from st2common.query.base import Querier, QueryContext -from st2common.runners import utils as runners_utils -from st2tests.config import parse_args -parse_args() - - -class QueryBaseTests(TestCase): - - @mock.patch.object( - Querier, - '_query_and_save_results', - mock.MagicMock(return_value=True) - ) - def test_fire_queries_doesnt_loop(self): - querier = Querier() - - mock_query_state_1 = QueryContext( - uuid.uuid4().hex, - uuid.uuid4().hex, - 'dummy_runner', - { - 'dummy': { - 'workflow_name': 'st2ci.st2_pkg_e2e_test', - 'execution_id': '6d624534-42ca-425c-aa3a-ccc676386fb2' - } - } - ) - - mock_query_state_2 = QueryContext( - uuid.uuid4().hex, - uuid.uuid4().hex, - 'dummy_runner', - { - 'dummy': { - 'workflow_name': 'st2ci.st2_pkg_e2e_test', - 'execution_id': '6d624534-42ca-425c-aa3a-ccc676386fb3' - } - } - ) - - mock_query_state_3 = QueryContext( - uuid.uuid4().hex, - uuid.uuid4().hex, - 'dummy_runner', - { - 'dummy': { - 'workflow_name': 'st2ci.st2_pkg_e2e_test', - 'execution_id': '6d624534-42ca-425c-aa3a-ccc676386fb4' - } - } - ) - - now = time.time() - query_contexts = six.moves.queue.Queue() - query_contexts.put((now + 100000, mock_query_state_1)), - query_contexts.put((now + 100001, mock_query_state_2)), - query_contexts.put((now - 200000, mock_query_state_3)), - querier._query_contexts = query_contexts - querier._fire_queries() - self.assertEqual(querier._query_contexts.qsize(), 2) - - @mock.patch.object( - Querier, - 'query', - mock.MagicMock(return_value=(action_constants.LIVEACTION_STATUS_RUNNING, {})) - ) - @mock.patch.object( - Querier, - '_update_action_results', - mock.MagicMock(return_value=None) - ) - @mock.patch.object( - Querier, - '_is_state_object_exist', - mock.MagicMock(return_value=True) - ) - @mock.patch.object( - Querier, - '_delete_state_object', - mock.MagicMock(return_value=None) - ) - def test_query_rescheduled(self): - querier = Querier() - - mock_query_state_1 = QueryContext( - uuid.uuid4().hex, - uuid.uuid4().hex, - 'dummy_runner', - { - 'dummy': { - 'workflow_name': 'st2ci.st2_pkg_e2e_test', - 'execution_id': '6d624534-42ca-425c-aa3a-ccc676386fb2' - } - } - ) - - now = time.time() - query_contexts = six.moves.queue.Queue() - query_contexts.put((now - 200000, mock_query_state_1)), - querier._query_contexts = query_contexts - querier._fire_queries(blocking=True) - self.assertFalse(Querier._delete_state_object.called) - self.assertEqual(querier._query_contexts.qsize(), 1) - - @mock.patch.object( - Querier, - 'query', - mock.MagicMock(return_value=(action_constants.LIVEACTION_STATUS_SUCCEEDED, {})) - ) - @mock.patch.object( - Querier, - '_update_action_results', - mock.MagicMock(return_value=None) - ) - @mock.patch.object( - Querier, - '_is_state_object_exist', - mock.MagicMock(return_value=True) - ) - @mock.patch.object( - Querier, - '_delete_state_object', - mock.MagicMock(return_value=None) - ) - @mock.patch.object( - runners_utils, - 'invoke_post_run', - mock.MagicMock(return_value=None) - ) - def test_query_completed(self): - querier = Querier() - - mock_query_state_1 = QueryContext( - uuid.uuid4().hex, - uuid.uuid4().hex, - 'dummy_runner', - { - 'dummy': { - 'workflow_name': 'st2ci.st2_pkg_e2e_test', - 'execution_id': '6d624534-42ca-425c-aa3a-ccc676386fb2' - } - } - ) - - now = time.time() - query_contexts = six.moves.queue.Queue() - query_contexts.put((now - 200000, mock_query_state_1)), - querier._query_contexts = query_contexts - querier._fire_queries(blocking=True) - self.assertTrue(runners_utils.invoke_post_run.called) - self.assertTrue(Querier._delete_state_object.called) - self.assertEqual(querier._query_contexts.qsize(), 0) - - @mock.patch.object( - Querier, - 'query', - mock.MagicMock(return_value=(action_constants.LIVEACTION_STATUS_RUNNING, {})) - ) - @mock.patch.object( - Querier, - '_update_action_results', - mock.MagicMock(return_value=None) - ) - @mock.patch.object( - Querier, - '_is_state_object_exist', - mock.MagicMock(return_value=False) - ) - @mock.patch.object( - Querier, - '_delete_state_object', - mock.MagicMock(return_value=None) - ) - def test_state_db_entry_deleted(self): - querier = Querier() - - mock_query_state_1 = QueryContext( - uuid.uuid4().hex, - uuid.uuid4().hex, - 'dummy_runner', - { - 'dummy': { - 'workflow_name': 'st2ci.st2_pkg_e2e_test', - 'execution_id': '6d624534-42ca-425c-aa3a-ccc676386fb2' - } - } - ) - - now = time.time() - query_contexts = six.moves.queue.Queue() - query_contexts.put((now - 200000, mock_query_state_1)), - querier._query_contexts = query_contexts - querier._fire_queries(blocking=True) - self.assertFalse(Querier._delete_state_object.called) - self.assertEqual(querier._query_contexts.qsize(), 0) diff --git a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/__init__.py b/st2tests/st2tests/fixtures/packs/runners/test_querymodule/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/callback/__init__.py b/st2tests/st2tests/fixtures/packs/runners/test_querymodule/callback/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/callback/test_querymodule.py b/st2tests/st2tests/fixtures/packs/runners/test_querymodule/callback/test_querymodule.py deleted file mode 100644 index 8d9e0847b7..0000000000 --- a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/callback/test_querymodule.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from st2common import log as logging -from st2common.callback import base as callback - - -LOG = logging.getLogger(__name__) - - -def get_instance(): - return MockRunnerCallbackHandler - - -class MockRunnerCallbackHandler(callback.AsyncActionExecutionCallbackHandler): - - @classmethod - def callback(cls, url, context, status, result): - pass diff --git a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/query/__init__.py b/st2tests/st2tests/fixtures/packs/runners/test_querymodule/query/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/query/test_querymodule.py b/st2tests/st2tests/fixtures/packs/runners/test_querymodule/query/test_querymodule.py deleted file mode 100644 index 9dc65625df..0000000000 --- a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/query/test_querymodule.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from st2common.query.base import Querier -from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED - - -class TestQuerier(Querier): - def __init__(self, *args, **kwargs): - super(TestQuerier, self).__init__(*args, **kwargs) - - def query(self, execution_id, query_context, last_query_time=None): - return (LIVEACTION_STATUS_SUCCEEDED, {'called_with': {execution_id: query_context}}) - - -def get_instance(): - return TestQuerier() From 03aafce2aca6b1cc84d6150267882b8e99f9cdc9 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Mon, 18 Jan 2021 23:17:10 +0100 Subject: [PATCH 22/31] Change expected number of runners on the tests --- st2api/tests/unit/controllers/v1/test_packs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/st2api/tests/unit/controllers/v1/test_packs.py b/st2api/tests/unit/controllers/v1/test_packs.py index 2fc64b1332..7286a9b204 100644 --- a/st2api/tests/unit/controllers/v1/test_packs.py +++ b/st2api/tests/unit/controllers/v1/test_packs.py @@ -540,15 +540,15 @@ def test_packs_register_endpoint(self, mock_get_packs): {'packs': ['dummy_pack_1'], 'types': ['action']}) self.assertEqual(resp.status_int, 200) - # 14 real plus 1 mock runner - self.assertEqual(resp.json, {'actions': 1, 'runners': 15}) + # 13 real plus 1 mock runner + self.assertEqual(resp.json, {'actions': 1, 'runners': 14}) # Verify that plural name form also works resp = self.app.post_json('/v1/packs/register', {'packs': ['dummy_pack_1'], 'types': ['actions']}) self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.json, {'actions': 1, 'runners': 15}) + self.assertEqual(resp.json, {'actions': 1, 'runners': 14}) # Register single resource from a single pack specified multiple times - verify that # resources from the same pack are only registered once @@ -558,7 +558,7 @@ def test_packs_register_endpoint(self, mock_get_packs): 'fail_on_failure': False}) self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.json, {'actions': 1, 'runners': 15}) + self.assertEqual(resp.json, {'actions': 1, 'runners': 14}) # Register resources from a single (non-existent pack) resp = self.app.post_json('/v1/packs/register', {'packs': ['doesntexist']}, From 8262088b27a685d2bd15169d5811b435d4c0b9cf Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Mon, 18 Jan 2021 23:47:52 +0100 Subject: [PATCH 23/31] Add comment to trigger another travis ci-unit run --- st2api/tests/unit/controllers/v1/test_packs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/st2api/tests/unit/controllers/v1/test_packs.py b/st2api/tests/unit/controllers/v1/test_packs.py index 7286a9b204..9406a50af0 100644 --- a/st2api/tests/unit/controllers/v1/test_packs.py +++ b/st2api/tests/unit/controllers/v1/test_packs.py @@ -548,6 +548,7 @@ def test_packs_register_endpoint(self, mock_get_packs): {'packs': ['dummy_pack_1'], 'types': ['actions']}) self.assertEqual(resp.status_int, 200) + # 13 real plus 1 mock runner self.assertEqual(resp.json, {'actions': 1, 'runners': 14}) # Register single resource from a single pack specified multiple times - verify that @@ -558,6 +559,7 @@ def test_packs_register_endpoint(self, mock_get_packs): 'fail_on_failure': False}) self.assertEqual(resp.status_int, 200) + # 13 real plus 1 mock runner self.assertEqual(resp.json, {'actions': 1, 'runners': 14}) # Register resources from a single (non-existent pack) From 2c7b29775b98d0391351b80f64a523afdb967b4a Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Tue, 19 Jan 2021 21:37:51 +0100 Subject: [PATCH 24/31] increase ci-unit command timeout --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ea4d350591..d021f082a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ env: - TASK=ci-packs-tests COMMAND_THRESHOLD=300 - - TASK=ci-unit COMMAND_THRESHOLD=300 + - TASK=ci-unit COMMAND_THRESHOLD=900 - TASK=ci-integration COMMAND_THRESHOLD=300 From facaa4e842acaecead953716f63e535ec98511df Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Wed, 20 Jan 2021 00:34:05 +0100 Subject: [PATCH 25/31] Remove tests for the query and callback module and start to delete the code (this is where I'll proceed tomorrow) --- st2common/st2common/constants/runners.py | 2 - st2common/st2common/runners/base.py | 45 ----------------------- st2common/st2common/util/loader.py | 2 - st2common/tests/unit/test_runners_base.py | 24 ------------ 4 files changed, 73 deletions(-) diff --git a/st2common/st2common/constants/runners.py b/st2common/st2common/constants/runners.py index 12fdc617f1..fe78a6497f 100644 --- a/st2common/st2common/constants/runners.py +++ b/st2common/st2common/constants/runners.py @@ -87,5 +87,3 @@ # Namespaces for dynamically loaded runner modules RUNNERS_NAMESPACE = "st2common.runners.runner" -RUNNERS_QUERY_MODULES_NAMESPACE = 'st2common.runners.query' -RUNNERS_CALLBACK_MODULES_NAMESPACE = 'st2common.runners.callback' diff --git a/st2common/st2common/runners/base.py b/st2common/st2common/runners/base.py index 6d7a1ade51..1358f5c812 100644 --- a/st2common/st2common/runners/base.py +++ b/st2common/st2common/runners/base.py @@ -29,8 +29,6 @@ from st2common.constants import action as action_constants from st2common.constants import pack as pack_constants from st2common.constants.runners import RUNNERS_NAMESPACE -from st2common.constants.runners import RUNNERS_QUERY_MODULES_NAMESPACE -from st2common.constants.runners import RUNNERS_CALLBACK_MODULES_NAMESPACE from st2common.content.utils import get_pack_directory from st2common.content.utils import get_pack_base_path from st2common.exceptions import actionrunner as exc @@ -55,9 +53,6 @@ 'get_runner', 'get_metadata', - - 'get_query_module', - 'get_callback_module' ] @@ -116,39 +111,6 @@ def get_runner_module(name): return module - -def get_query_module(name): - """ - Retrieve runner query module for the provided runner. - """ - # NOTE: For backward compatibility we also support "_" in place of "-" - from stevedore.exception import NoMatches - - try: - module = get_plugin_instance(RUNNERS_QUERY_MODULES_NAMESPACE, name, invoke_on_load=False) - except NoMatches: - name = name.replace('_', '-') - module = get_plugin_instance(RUNNERS_QUERY_MODULES_NAMESPACE, name, invoke_on_load=False) - - return module - - -def get_callback_module(name): - """ - Retrieve runner callback module for the provided runner. - """ - # NOTE: For backward compatibility we also support "_" in place of "-" - from stevedore.exception import NoMatches - - try: - module = get_plugin_instance(RUNNERS_CALLBACK_MODULES_NAMESPACE, name, invoke_on_load=False) - except NoMatches: - name = name.replace('_', '-') - module = get_plugin_instance(RUNNERS_CALLBACK_MODULES_NAMESPACE, name, invoke_on_load=False) - - return module - - def get_metadata(package_name): """ Return runner related metadata for the provided runner package name. @@ -190,7 +152,6 @@ def __init__(self, runner_id): self.entry_point = None self.libs_dir_path = None self.context = None - self.callback = None self.auth_token = None self.rerun_ex_ref = None @@ -227,12 +188,6 @@ def cancel(self): self.liveaction.context ) - def post_run(self, status, result): - if self.callback and isinstance(self.callback, dict) and 'source' in self.callback: - callback_module = get_callback_module(self.callback['source']) - callback_handler = callback_module.get_instance() - callback_handler.callback(self.liveaction) - @deprecated def get_pack_name(self): return self.get_pack_ref() diff --git a/st2common/st2common/util/loader.py b/st2common/st2common/util/loader.py index 3acdd1e090..0e27a0da32 100644 --- a/st2common/st2common/util/loader.py +++ b/st2common/st2common/util/loader.py @@ -44,8 +44,6 @@ # Cache for dynamically loaded runner modules RUNNER_MODULES_CACHE = defaultdict(dict) -QUERIER_MODULES_CACHE = {} -CALLBACK_MODULES_CACHE = {} def _register_plugin_path(plugin_dir_abs_path): diff --git a/st2common/tests/unit/test_runners_base.py b/st2common/tests/unit/test_runners_base.py index 54d13ee18d..0445bdfd7b 100644 --- a/st2common/tests/unit/test_runners_base.py +++ b/st2common/tests/unit/test_runners_base.py @@ -18,8 +18,6 @@ from stevedore.exception import NoMatches from st2common.runners.base import get_runner -from st2common.runners.base import get_query_module -from st2common.runners.base import get_callback_module from st2common.exceptions.actionrunner import ActionRunnerCreateError from st2tests.base import DbTestCase @@ -35,25 +33,3 @@ def test_get_runner_failure_not_found(self): expected_msg = 'Failed to find runner invalid-name-not-found.*' self.assertRaisesRegexp(ActionRunnerCreateError, expected_msg, get_runner, 'invalid-name-not-found') - - def test_get_query_module_success(self): - query_module = get_query_module('mock_query_callback') - - self.assertEqual(query_module.__name__, 'mock_query_callback.query') - self.assertTrue(query_module.get_instance()) - - def test_get_query_module_failure_not_found(self): - expected_msg = 'No .*? driver found.*' - self.assertRaisesRegexp(NoMatches, expected_msg, - get_query_module, 'invalid-name-not-found') - - def test_get_callback_module_success(self): - callback_module = get_callback_module('mock_query_callback') - - self.assertEqual(callback_module.__name__, 'mock_query_callback.callback') - self.assertTrue(callback_module.get_instance()) - - def test_get_callback_module_failure_not_found(self): - expected_msg = 'No .*? driver found.*' - self.assertRaisesRegexp(NoMatches, expected_msg, - get_callback_module, 'invalid-name-not-found') From 51c601711816def57ff4bbf7f88192606b3f2a4f Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Wed, 20 Jan 2021 22:09:58 +0100 Subject: [PATCH 26/31] Add the post_run method for the base runner again --- st2common/st2common/runners/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/st2common/st2common/runners/base.py b/st2common/st2common/runners/base.py index 1358f5c812..4470334ff5 100644 --- a/st2common/st2common/runners/base.py +++ b/st2common/st2common/runners/base.py @@ -188,6 +188,9 @@ def cancel(self): self.liveaction.context ) + def post_run(self, status, result): + pass + @deprecated def get_pack_name(self): return self.get_pack_ref() From fbfd8bd2f8cb8fc1f2972b6badba1f9a691724b3 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Wed, 20 Jan 2021 22:31:23 +0100 Subject: [PATCH 27/31] Remove tests for the async action runners --- .../tests/unit/test_runner_container.py | 81 ------------------- 1 file changed, 81 deletions(-) diff --git a/st2actions/tests/unit/test_runner_container.py b/st2actions/tests/unit/test_runner_container.py index 3ccfb7a4ea..cc12e7ebc0 100644 --- a/st2actions/tests/unit/test_runner_container.py +++ b/st2actions/tests/unit/test_runner_container.py @@ -330,87 +330,6 @@ def test_dispatch_override_default_action_params(self): self.assertTrue(result.get('action_params').get('actionint') == 20) self.assertTrue(result.get('action_params').get('actionstr') == 'foo') - def test_state_db_created_for_polling_async_actions(self): - runner_container = get_runner_container() - - params = { - 'actionstr': 'foo', - 'actionint': 20, - 'async_test': True - } - - liveaction_db = self._get_liveaction_model( - RunnerContainerTest.polling_async_action_db, - params - ) - - liveaction_db = LiveAction.add_or_update(liveaction_db) - executions.create_execution_object(liveaction_db) - - # Assert that execution ran without exceptions. - with mock.patch('st2actions.container.base.get_runner', - mock.Mock(return_value=polling_async_runner.get_runner())): - runner_container.dispatch(liveaction_db) - states = ActionExecutionState.get_all() - found = [state for state in states if state.execution_id == liveaction_db.id] - - self.assertTrue(len(found) > 0, 'There should be a state db object.') - self.assertTrue(len(found) == 1, 'There should only be one state db object.') - self.assertIsNotNone(found[0].query_context) - self.assertIsNotNone(found[0].query_module) - - @mock.patch.object( - PollingAsyncActionRunner, - 'is_polling_enabled', - mock.MagicMock(return_value=False)) - def test_state_db_not_created_for_disabled_polling_async_actions(self): - runner_container = get_runner_container() - - params = { - 'actionstr': 'foo', - 'actionint': 20, - 'async_test': True - } - - liveaction_db = self._get_liveaction_model( - RunnerContainerTest.polling_async_action_db, - params - ) - - liveaction_db = LiveAction.add_or_update(liveaction_db) - executions.create_execution_object(liveaction_db) - - # Assert that execution ran without exceptions. - runner_container.dispatch(liveaction_db) - states = ActionExecutionState.get_all() - found = [state for state in states if state.execution_id == liveaction_db.id] - - self.assertTrue(len(found) == 0, 'There should not be a state db object.') - - def test_state_db_not_created_for_async_actions(self): - runner_container = get_runner_container() - - params = { - 'actionstr': 'foo', - 'actionint': 20, - 'async_test': True - } - - liveaction_db = self._get_liveaction_model( - RunnerContainerTest.async_action_db, - params - ) - - liveaction_db = LiveAction.add_or_update(liveaction_db) - executions.create_execution_object(liveaction_db) - - # Assert that execution ran without exceptions. - runner_container.dispatch(liveaction_db) - states = ActionExecutionState.get_all() - found = [state for state in states if state.execution_id == liveaction_db.id] - - self.assertTrue(len(found) == 0, 'There should not be a state db object.') - def _get_liveaction_model(self, action_db, params): status = action_constants.LIVEACTION_STATUS_REQUESTED start_timestamp = date_utils.get_datetime_utc_now() From f5ad9e3b79269c0c09876d97a1c1d546d0301d7c Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 21 Jan 2021 10:10:27 +0100 Subject: [PATCH 28/31] Fixes to pass lint checks --- st2actions/tests/unit/test_runner_container.py | 3 --- st2common/st2common/runners/base.py | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/st2actions/tests/unit/test_runner_container.py b/st2actions/tests/unit/test_runner_container.py index cc12e7ebc0..31a13afa2d 100644 --- a/st2actions/tests/unit/test_runner_container.py +++ b/st2actions/tests/unit/test_runner_container.py @@ -26,8 +26,6 @@ from st2common.models.db.liveaction import LiveActionDB from st2common.models.db.runner import RunnerTypeDB from st2common.persistence.liveaction import LiveAction -from st2common.persistence.executionstate import ActionExecutionState -from st2common.runners.base import PollingAsyncActionRunner from st2common.services import executions from st2common.util import date as date_utils from st2common.transport.publishers import PoolPublisher @@ -71,7 +69,6 @@ } from st2tests.mocks.runners import runner -from st2tests.mocks.runners import polling_async_runner @mock.patch('st2common.runners.base.get_runner', mock.Mock(return_value=runner.get_runner())) diff --git a/st2common/st2common/runners/base.py b/st2common/st2common/runners/base.py index 4470334ff5..6a9656b9a1 100644 --- a/st2common/st2common/runners/base.py +++ b/st2common/st2common/runners/base.py @@ -111,6 +111,7 @@ def get_runner_module(name): return module + def get_metadata(package_name): """ Return runner related metadata for the provided runner package name. From bc43cdfc0cc1ae8014cc6f999a629c3ee0677a5a Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 21 Jan 2021 15:32:40 +0100 Subject: [PATCH 29/31] fix remaining lint issue --- st2common/tests/unit/test_runners_base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/st2common/tests/unit/test_runners_base.py b/st2common/tests/unit/test_runners_base.py index 0445bdfd7b..34ede41adf 100644 --- a/st2common/tests/unit/test_runners_base.py +++ b/st2common/tests/unit/test_runners_base.py @@ -15,8 +15,6 @@ from __future__ import absolute_import -from stevedore.exception import NoMatches - from st2common.runners.base import get_runner from st2common.exceptions.actionrunner import ActionRunnerCreateError From 34ffaf45622d131e464989359f6bf48f5f5d833f Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 22 Jan 2021 19:24:56 +0100 Subject: [PATCH 30/31] Add tests for the async action runner again --- .../tests/unit/test_runner_container.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/st2actions/tests/unit/test_runner_container.py b/st2actions/tests/unit/test_runner_container.py index 31a13afa2d..3ccfb7a4ea 100644 --- a/st2actions/tests/unit/test_runner_container.py +++ b/st2actions/tests/unit/test_runner_container.py @@ -26,6 +26,8 @@ from st2common.models.db.liveaction import LiveActionDB from st2common.models.db.runner import RunnerTypeDB from st2common.persistence.liveaction import LiveAction +from st2common.persistence.executionstate import ActionExecutionState +from st2common.runners.base import PollingAsyncActionRunner from st2common.services import executions from st2common.util import date as date_utils from st2common.transport.publishers import PoolPublisher @@ -69,6 +71,7 @@ } from st2tests.mocks.runners import runner +from st2tests.mocks.runners import polling_async_runner @mock.patch('st2common.runners.base.get_runner', mock.Mock(return_value=runner.get_runner())) @@ -327,6 +330,87 @@ def test_dispatch_override_default_action_params(self): self.assertTrue(result.get('action_params').get('actionint') == 20) self.assertTrue(result.get('action_params').get('actionstr') == 'foo') + def test_state_db_created_for_polling_async_actions(self): + runner_container = get_runner_container() + + params = { + 'actionstr': 'foo', + 'actionint': 20, + 'async_test': True + } + + liveaction_db = self._get_liveaction_model( + RunnerContainerTest.polling_async_action_db, + params + ) + + liveaction_db = LiveAction.add_or_update(liveaction_db) + executions.create_execution_object(liveaction_db) + + # Assert that execution ran without exceptions. + with mock.patch('st2actions.container.base.get_runner', + mock.Mock(return_value=polling_async_runner.get_runner())): + runner_container.dispatch(liveaction_db) + states = ActionExecutionState.get_all() + found = [state for state in states if state.execution_id == liveaction_db.id] + + self.assertTrue(len(found) > 0, 'There should be a state db object.') + self.assertTrue(len(found) == 1, 'There should only be one state db object.') + self.assertIsNotNone(found[0].query_context) + self.assertIsNotNone(found[0].query_module) + + @mock.patch.object( + PollingAsyncActionRunner, + 'is_polling_enabled', + mock.MagicMock(return_value=False)) + def test_state_db_not_created_for_disabled_polling_async_actions(self): + runner_container = get_runner_container() + + params = { + 'actionstr': 'foo', + 'actionint': 20, + 'async_test': True + } + + liveaction_db = self._get_liveaction_model( + RunnerContainerTest.polling_async_action_db, + params + ) + + liveaction_db = LiveAction.add_or_update(liveaction_db) + executions.create_execution_object(liveaction_db) + + # Assert that execution ran without exceptions. + runner_container.dispatch(liveaction_db) + states = ActionExecutionState.get_all() + found = [state for state in states if state.execution_id == liveaction_db.id] + + self.assertTrue(len(found) == 0, 'There should not be a state db object.') + + def test_state_db_not_created_for_async_actions(self): + runner_container = get_runner_container() + + params = { + 'actionstr': 'foo', + 'actionint': 20, + 'async_test': True + } + + liveaction_db = self._get_liveaction_model( + RunnerContainerTest.async_action_db, + params + ) + + liveaction_db = LiveAction.add_or_update(liveaction_db) + executions.create_execution_object(liveaction_db) + + # Assert that execution ran without exceptions. + runner_container.dispatch(liveaction_db) + states = ActionExecutionState.get_all() + found = [state for state in states if state.execution_id == liveaction_db.id] + + self.assertTrue(len(found) == 0, 'There should not be a state db object.') + def _get_liveaction_model(self, action_db, params): status = action_constants.LIVEACTION_STATUS_REQUESTED start_timestamp = date_utils.get_datetime_utc_now() From cf54ec5da3197ffea8f5af196e979e9974a4d933 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 22 Jan 2021 20:35:35 +0100 Subject: [PATCH 31/31] increase timeout for the integration tests on travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4a94339332..ebb2e9bc94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,7 @@ env: # - TASK=ci-packs-tests COMMAND_THRESHOLD=300 # - TASK=ci-unit COMMAND_THRESHOLD=300 - - TASK=ci-integration COMMAND_THRESHOLD=300 + - TASK=ci-integration COMMAND_THRESHOLD=600 # jobs: # fast_finish: true