diff --git a/.travis.yml b/.travis.yml index 4a94339332..ebb2e9bc94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,7 @@ env: # - TASK=ci-packs-tests COMMAND_THRESHOLD=300 # - TASK=ci-unit COMMAND_THRESHOLD=300 - - TASK=ci-integration COMMAND_THRESHOLD=300 + - TASK=ci-integration COMMAND_THRESHOLD=600 # jobs: # fast_finish: true diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1afaf9dff4..9840a4ee19 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -61,10 +61,14 @@ Removed * Removed submit-debug-info tool and the st2debug component #5103 * Removed check-licence script (cleanup) #5092 + Contributed by @kroustou * Updated Makefile and CI to use Python 3 only, removing Python 2 (cleanup) #5090 Contributed by @blag +* Remove st2resultstracker from st2ctl, the development environment and the st2actions setup.py (cleanup) #5108 + + Contributed by @winem 3.3.0 - October 06, 2020 ------------------------ diff --git a/conf/HA/st2.conf.sample b/conf/HA/st2.conf.sample index 20cbd422ff..b52fe81bf4 100644 --- a/conf/HA/st2.conf.sample +++ b/conf/HA/st2.conf.sample @@ -23,9 +23,6 @@ logging = /etc/st2/logging.actionrunner.conf # The line should be commented and 'always-copy' removed when using EL7 or EL8 as it causes virtualenv issues on pack install virtualenv_opts = --always-copy -[resultstracker] -logging = /etc/st2/logging.resultstracker.conf - [notifier] logging = /etc/st2/logging.notifier.conf diff --git a/conf/logrotate.conf b/conf/logrotate.conf index 7940fead0e..c5f60005d6 100644 --- a/conf/logrotate.conf +++ b/conf/logrotate.conf @@ -71,15 +71,6 @@ notifempty endscript } -## Results Tracker -/var/log/st2/st2resultstracker*.log { - daily - rotate 5 - postrotate - st2ctl reopen-log-files st2resultstracker > /dev/null - endscript -} - ## Notifier /var/log/st2/st2notifier*.log { daily diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index a8111f59af..758b743e75 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -224,18 +224,6 @@ logging = /etc/st2/logging.notifier.conf # Enable/Disable support for pack common libs. Setting this config to ``True`` would allow you to place common library code for sensors and actions in lib/ folder in packs and use them in python sensors and actions. See https://docs.stackstorm.com/reference/sharing_code_sensors_actions.html for details. enable_common_libs = False -[resultstracker] -# Sleep delay in between queries when query queue is empty. -empty_q_sleep_time = 1 -# Location of the logging configuration file. -logging = /etc/st2/logging.resultstracker.conf -# Sleep delay for query when there is no more worker in pool. -no_workers_sleep_time = 1 -# Time interval between queries to external workflow system. -query_interval = 5 -# Number of threads to use to query external workflow systems. -thread_pool_size = 10 - [rulesengine] # Location of the logging configuration file. logging = /etc/st2/logging.rulesengine.conf diff --git a/conf/st2.dev.conf b/conf/st2.dev.conf index 08aa45870d..2357b08263 100644 --- a/conf/st2.dev.conf +++ b/conf/st2.dev.conf @@ -114,10 +114,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [scheduler] logging = st2actions/conf/logging.scheduler.conf diff --git a/conf/st2.package.conf b/conf/st2.package.conf index 661d877834..dbd10b211b 100644 --- a/conf/st2.package.conf +++ b/conf/st2.package.conf @@ -22,9 +22,6 @@ logging = /etc/st2/logging.rulesengine.conf logging = /etc/st2/logging.actionrunner.conf virtualenv_opts = --always-copy -[resultstracker] -logging = /etc/st2/logging.resultstracker.conf - [notifier] logging = /etc/st2/logging.notifier.conf diff --git a/conf/st2.tests.conf b/conf/st2.tests.conf index 87c97595e4..0e9afeb288 100644 --- a/conf/st2.tests.conf +++ b/conf/st2.tests.conf @@ -84,10 +84,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/conf/st2.tests1.conf b/conf/st2.tests1.conf index 2ec4592807..53ae62e5ae 100644 --- a/conf/st2.tests1.conf +++ b/conf/st2.tests1.conf @@ -67,9 +67,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/conf/st2.tests2.conf b/conf/st2.tests2.conf index 1536789144..cb7be38a71 100644 --- a/conf/st2.tests2.conf +++ b/conf/st2.tests2.conf @@ -89,10 +89,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/st2actions/bin/st2resultstracker b/st2actions/bin/st2resultstracker deleted file mode 100755 index 0dae3ad493..0000000000 --- a/st2actions/bin/st2resultstracker +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the StackStorm, Inc ('StackStorm') under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -from st2actions.cmd import st2resultstracker - - -if __name__ == '__main__': - sys.exit(st2resultstracker.main()) diff --git a/st2actions/conf/logging.resultstracker.conf b/st2actions/conf/logging.resultstracker.conf deleted file mode 100644 index 67551477ff..0000000000 --- a/st2actions/conf/logging.resultstracker.conf +++ /dev/null @@ -1,44 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=consoleHandler, fileHandler, auditHandler - -[formatters] -keys=simpleConsoleFormatter, verboseConsoleFormatter, gelfFormatter - -[logger_root] -level=DEBUG -handlers=consoleHandler, fileHandler, auditHandler - -[handler_consoleHandler] -class=StreamHandler -level=INFO -formatter=simpleConsoleFormatter -args=(sys.stdout,) - -[handler_fileHandler] -class=st2common.log.FormatNamedFileHandler -level=DEBUG -formatter=verboseConsoleFormatter -args=('logs/st2resultstracker.log',) - -[handler_auditHandler] -class=st2common.log.FormatNamedFileHandler -level=AUDIT -formatter=gelfFormatter -args=('logs/st2resultstracker.audit.log',) - -[formatter_simpleConsoleFormatter] -class=st2common.logging.formatters.ConsoleLogFormatter -format=%(asctime)s %(levelname)s [-] %(message)s -datefmt= - -[formatter_verboseConsoleFormatter] -class=st2common.logging.formatters.ConsoleLogFormatter -format=%(asctime)s %(thread)s %(levelname)s %(module)s [-] %(message)s -datefmt= - -[formatter_gelfFormatter] -class=st2common.logging.formatters.GelfLogFormatter -format=%(message)s diff --git a/st2actions/conf/syslog.resultstracker.conf b/st2actions/conf/syslog.resultstracker.conf deleted file mode 100644 index 2196a44377..0000000000 --- a/st2actions/conf/syslog.resultstracker.conf +++ /dev/null @@ -1,22 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=syslogHandler - -[formatters] -keys=syslogVerboseFormatter - -[logger_root] -level=DEBUG -handlers=syslogHandler - -[handler_syslogHandler] -class=st2common.log.ConfigurableSyslogHandler -level=DEBUG -formatter=syslogVerboseFormatter -args=() - -[formatter_syslogVerboseFormatter] -format=st2resultstracker[%(process)d]: %(levelname)s %(thread)s %(module)s [-] %(message)s -datefmt= diff --git a/st2actions/setup.py b/st2actions/setup.py index 4845eb83d9..6fcb2cde92 100644 --- a/st2actions/setup.py +++ b/st2actions/setup.py @@ -47,7 +47,6 @@ scripts=[ 'bin/st2actionrunner', 'bin/st2notifier', - 'bin/st2resultstracker', 'bin/st2workflowengine', 'bin/st2scheduler', ] diff --git a/st2actions/st2actions/cmd/st2resultstracker.py b/st2actions/st2actions/cmd/st2resultstracker.py deleted file mode 100644 index 69d4348055..0000000000 --- a/st2actions/st2actions/cmd/st2resultstracker.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -from st2common.util.monkey_patch import monkey_patch -monkey_patch() - -import os -import sys - -from st2common import log as logging -from st2common.service_setup import setup as common_setup -from st2common.service_setup import teardown as common_teardown -from st2actions.resultstracker import config -from st2actions.resultstracker import resultstracker - -__all__ = [ - 'main' -] - - -LOG = logging.getLogger(__name__) - - -def _setup(): - capabilities = { - 'name': 'resultstracker', - 'type': 'passive' - } - common_setup(service='resultstracker', config=config, setup_db=True, register_mq_exchanges=True, - register_signal_handlers=True, service_registry=True, capabilities=capabilities) - - -def _run_worker(): - LOG.info('(PID=%s) Results tracker started.', os.getpid()) - tracker = resultstracker.get_tracker() - try: - tracker.start(wait=True) - except (KeyboardInterrupt, SystemExit): - LOG.info('(PID=%s) Results tracker stopped.', os.getpid()) - tracker.shutdown() - except: - return 1 - return 0 - - -def _teardown(): - common_teardown() - - -def main(): - try: - _setup() - return _run_worker() - except SystemExit as exit_code: - sys.exit(exit_code) - except: - LOG.exception('(PID=%s) Results tracker quit due to exception.', os.getpid()) - return 1 - finally: - _teardown() diff --git a/st2actions/st2actions/resultstracker/__init__.py b/st2actions/st2actions/resultstracker/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2actions/st2actions/resultstracker/config.py b/st2actions/st2actions/resultstracker/config.py deleted file mode 100644 index 0e48c48dd4..0000000000 --- a/st2actions/st2actions/resultstracker/config.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -from oslo_config import cfg - -from st2common import config as common_config -from st2common.constants.system import VERSION_STRING -from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH - -common_config.register_opts() - -CONF = cfg.CONF - - -def parse_args(args=None): - cfg.CONF(args=args, version=VERSION_STRING, - default_config_files=[DEFAULT_CONFIG_FILE_PATH]) - - -def register_opts(): - _register_common_opts() - _register_results_tracker_opts() - - -def get_logging_config_path(): - return cfg.CONF.resultstracker.logging - - -def _register_common_opts(): - common_config.register_opts() - - -def _register_results_tracker_opts(): - resultstracker_opts = [ - cfg.StrOpt( - 'logging', default='/etc/st2/logging.resultstracker.conf', - help='Location of the logging configuration file.') - ] - - CONF.register_opts(resultstracker_opts, group='resultstracker') - - -register_opts() diff --git a/st2actions/st2actions/resultstracker/resultstracker.py b/st2actions/st2actions/resultstracker/resultstracker.py deleted file mode 100644 index dd8b618181..0000000000 --- a/st2actions/st2actions/resultstracker/resultstracker.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -import eventlet -import six - -from collections import defaultdict - -from st2common.query.base import QueryContext -from st2common import log as logging -from st2common.models.db.executionstate import ActionExecutionStateDB -from st2common.persistence.executionstate import ActionExecutionState -from st2common.transport import consumers -from st2common.transport import utils as transport_utils -from st2common.runners.base import get_query_module -from st2common.transport.queues import RESULTSTRACKER_ACTIONSTATE_WORK_QUEUE - -__all__ = [ - 'ResultsTracker', - 'get_tracker' -] - - -LOG = logging.getLogger(__name__) - - -class ResultsTracker(consumers.MessageHandler): - message_type = ActionExecutionStateDB - - def __init__(self, connection, queues): - super(ResultsTracker, self).__init__(connection, queues) - self._queriers = {} - self._query_threads = [] - self._failed_imports = set() - - def start(self, wait=False): - self._bootstrap() - super(ResultsTracker, self).start(wait=wait) - - def wait(self): - super(ResultsTracker, self).wait() - for thread in self._query_threads: - thread.wait() - - def shutdown(self): - super(ResultsTracker, self).shutdown() - LOG.info('Stats from queriers:') - self._print_stats() - - def _print_stats(self): - for _, querier in six.iteritems(self._queriers): - if querier: - querier.print_stats() - - def _bootstrap(self): - all_states = ActionExecutionState.get_all() - LOG.info('Found %d pending states in db.' % len(all_states)) - - query_contexts_dict = defaultdict(list) - for state_db in all_states: - try: - context = QueryContext.from_model(state_db) - except: - LOG.exception('Invalid state object: %s', state_db) - continue - query_module_name = state_db.query_module - querier = self.get_querier(query_module_name) - - if querier is not None: - query_contexts_dict[querier].append(context) - - for querier, contexts in six.iteritems(query_contexts_dict): - LOG.info('Found %d pending actions for query module %s', len(contexts), querier) - querier.add_queries(query_contexts=contexts) - - def process(self, query_context): - querier = self.get_querier(query_context.query_module) - context = QueryContext.from_model(query_context) - querier.add_queries(query_contexts=[context]) - return - - def get_querier(self, query_module_name): - if (query_module_name not in self._queriers and - query_module_name not in self._failed_imports): - try: - query_module = get_query_module(query_module_name) - except: - LOG.exception('Failed importing query module: %s', query_module_name) - self._failed_imports.add(query_module_name) - self._queriers[query_module_name] = None - else: - querier = query_module.get_instance() - self._queriers[query_module_name] = querier - self._query_threads.append(eventlet.spawn(querier.start)) - - return self._queriers[query_module_name] - - -def get_tracker(): - with transport_utils.get_connection() as conn: - return ResultsTracker(conn, [RESULTSTRACKER_ACTIONSTATE_WORK_QUEUE]) diff --git a/st2actions/tests/integration/test_action_state_consumer.py b/st2actions/tests/integration/test_action_state_consumer.py deleted file mode 100644 index 4cb0d05db0..0000000000 --- a/st2actions/tests/integration/test_action_state_consumer.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -import os -import imp - -import mock - -from st2common.transport.queues import RESULTSTRACKER_ACTIONSTATE_WORK_QUEUE -from st2actions.resultstracker.resultstracker import ResultsTracker -from st2common.models.db.executionstate import ActionExecutionStateDB -from st2common.persistence.executionstate import ActionExecutionState -from st2common.transport import utils as transport_utils -from st2tests.base import DbTestCase, EventletTestCase -from st2tests.fixturesloader import FixturesLoader -from st2tests.fixtures.packs.runners.test_querymodule.query.test_querymodule import TestQuerier - -FIXTURES_PACK = 'generic' -FIXTURES = {'liveactions': ['liveaction1.yaml']} -loader = FixturesLoader() - -CURRENT_DIR = os.path.dirname(__file__) -ST2CONTENT_DIR = os.path.join(CURRENT_DIR, '../../../st2tests/st2tests/fixtures/packs/runners') - -MOCK_RUNNER_NAME = 'test_querymodule' - -MOCK_QUERIER_PATH = '{0}/{1}/query/{1}.py'.format(ST2CONTENT_DIR, MOCK_RUNNER_NAME) -MOCK_QUERIER_PATH = os.path.abspath(MOCK_QUERIER_PATH) -MOCK_QUERIER_MODULE = imp.load_source(MOCK_RUNNER_NAME + '.query', MOCK_QUERIER_PATH) - - -@mock.patch('st2common.runners.base.get_query_module', - mock.Mock(return_value=MOCK_QUERIER_MODULE)) -@mock.patch('st2actions.resultstracker.resultstracker.get_query_module', - mock.Mock(return_value=MOCK_QUERIER_MODULE)) -class ActionStateConsumerTests(EventletTestCase, DbTestCase): - models = None - liveactions = None - - @classmethod - def setUpClass(cls): - super(ActionStateConsumerTests, cls).setUpClass() - DbTestCase.setUpClass() - cls.models = loader.save_fixtures_to_db(fixtures_pack=FIXTURES_PACK, - fixtures_dict=FIXTURES) - cls.liveactions = cls.models['liveactions'] - - @mock.patch.object(TestQuerier, 'query', mock.MagicMock(return_value=(False, {}))) - def test_process_message(self): - with transport_utils.get_connection() as conn: - tracker = ResultsTracker(conn, [RESULTSTRACKER_ACTIONSTATE_WORK_QUEUE]) - tracker._bootstrap() - state = ActionStateConsumerTests.get_state( - ActionStateConsumerTests.liveactions['liveaction1.yaml']) - tracker._queue_consumer._process_message(state) - querier = tracker.get_querier('test_querymodule') - self.assertEqual(querier._query_contexts.qsize(), 1) - - @classmethod - def get_state(cls, exec_db): - state = ActionExecutionStateDB(execution_id=str(exec_db.id), query_context={'id': 'foo'}, - query_module='test_querymodule') - return ActionExecutionState.add_or_update(state) - - @classmethod - def tearDownClass(cls): - loader.delete_models_from_db(ActionStateConsumerTests.models) diff --git a/st2actions/tests/integration/test_resultstracker.py b/st2actions/tests/integration/test_resultstracker.py deleted file mode 100644 index d5b9a0422d..0000000000 --- a/st2actions/tests/integration/test_resultstracker.py +++ /dev/null @@ -1,378 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -import os -import imp - -import eventlet -import mock - -from oslo_config import cfg - -from st2common.constants import action as action_constants -from st2common.exceptions.db import StackStormDBObjectNotFoundError -from st2common.persistence.action import Action -from st2common.persistence.executionstate import ActionExecutionState -from st2common.persistence.liveaction import LiveAction -from st2common.runners import utils as runners_utils -from st2common.services import executions -from st2common.util import action_db as action_db_utils -from st2tests.base import (DbTestCase, EventletTestCase) -from st2tests.fixturesloader import FixturesLoader - -__all__ = [ - 'ResultsTrackerTests' -] - - -FIXTURES_LOADER = FixturesLoader() -FIXTURES_PACK = 'generic' -FIXTURES = { - 'actionstates': ['state1.yaml', 'state2.yaml'], - 'liveactions': ['liveaction1.yaml', 'liveaction2.yaml'] -} - -CURRENT_DIR = os.path.dirname(__file__) -ST2CONTENT_DIR = os.path.join(CURRENT_DIR, '../../../st2tests/st2tests/fixtures/packs/runners') - -MOCK_RUNNER_NAME = 'test_querymodule' - -MOCK_QUERIER_PATH = '{0}/{1}/query/{1}.py'.format(ST2CONTENT_DIR, MOCK_RUNNER_NAME) -MOCK_QUERIER_PATH = os.path.abspath(MOCK_QUERIER_PATH) -MOCK_QUERIER_MODULE = imp.load_source(MOCK_RUNNER_NAME + '.query', MOCK_QUERIER_PATH) - -MOCK_CALLBACK_PATH = '{0}/{1}/callback/{1}.py'.format(ST2CONTENT_DIR, MOCK_RUNNER_NAME) -MOCK_CALLBACK_PATH = os.path.abspath(MOCK_CALLBACK_PATH) -MOCK_CALLBACK_MODULE = imp.load_source(MOCK_RUNNER_NAME + '.callback', MOCK_CALLBACK_PATH) - - -@mock.patch.object( - executions, 'update_execution', mock.MagicMock(return_value=None)) -@mock.patch.object( - LiveAction, 'publish_update', mock.MagicMock(return_value=None)) -@mock.patch('st2common.runners.base.get_query_module', - mock.Mock(return_value=MOCK_QUERIER_MODULE)) -@mock.patch('st2actions.resultstracker.resultstracker.get_query_module', - mock.Mock(return_value=MOCK_QUERIER_MODULE)) -class ResultsTrackerTests(EventletTestCase, DbTestCase): - states = None - models = None - liveactions = None - - @classmethod - def setUpClass(cls): - super(ResultsTrackerTests, cls).setUpClass() - cfg.CONF.set_default('empty_q_sleep_time', 0.2, group='resultstracker') - cfg.CONF.set_default('no_workers_sleep_time', 0.1, group='resultstracker') - - @classmethod - def tearDownClass(cls): - cfg.CONF.set_default('empty_q_sleep_time', 1, group='resultstracker') - cfg.CONF.set_default('no_workers_sleep_time', 1, group='resultstracker') - super(ResultsTrackerTests, cls).tearDownClass() - - def setUp(self): - super(ResultsTrackerTests, self).setUp() - DbTestCase.setUpClass() - ResultsTrackerTests.models = FIXTURES_LOADER.save_fixtures_to_db( - fixtures_pack=FIXTURES_PACK, fixtures_dict=FIXTURES) - ResultsTrackerTests.states = ResultsTrackerTests.models['actionstates'] - ResultsTrackerTests.liveactions = ResultsTrackerTests.models['liveactions'] - ResultsTrackerTests._update_state_models() - - def tearDown(self): - FIXTURES_LOADER.delete_models_from_db(ResultsTrackerTests.models) - super(ResultsTrackerTests, self).tearDown() - - @mock.patch.object( - Action, 'get_by_ref', mock.MagicMock(return_value='foobar')) - def test_query_process(self): - tracker = self._get_tracker() - runners_utils.invoke_post_run = mock.Mock(return_value=None) - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - # Process the state objects. - tracker._bootstrap() - eventlet.sleep(1) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertIsNotNone(exec_db.result['called_with'][exec_id], exec_db.result) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertIsNotNone(exec_db.result['called_with'][exec_id], exec_db.result) - - tracker.shutdown() - - # Ensure state objects are deleted. - self.assertRaises( - StackStormDBObjectNotFoundError, - ActionExecutionState.get_by_id, - ResultsTrackerTests.states['state1.yaml'].id - ) - - self.assertRaises( - StackStormDBObjectNotFoundError, - ActionExecutionState.get_by_id, - ResultsTrackerTests.states['state2.yaml'].id - ) - - # Ensure invoke_post_run is called. - self.assertEqual(2, runners_utils.invoke_post_run.call_count) - - def test_start_shutdown(self): - tracker = self._get_tracker() - tracker.start() - eventlet.sleep(0.1) - tracker.shutdown() - - def test_get_querier_success(self): - tracker = self._get_tracker() - self.assertIsNotNone(tracker.get_querier('test_querymodule')) - - def test_get_querier_not_found(self): - with mock.patch('st2actions.resultstracker.resultstracker.get_query_module', - mock.Mock(side_effect=Exception('Not found'))): - tracker = self._get_tracker() - self.assertEqual(tracker.get_querier('this_module_aint_exist'), None) - - def test_querier_started(self): - tracker = self._get_tracker() - querier = tracker.get_querier('test_querymodule') - eventlet.sleep(0.1) - self.assertTrue(querier.is_started(), 'querier must have been started.') - - def test_delete_state_object_on_error_at_query(self): - tracker = self._get_tracker() - querier = tracker.get_querier('test_querymodule') - querier._delete_state_object = mock.Mock(return_value=None) - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - with mock.patch.object( - querier.__class__, 'query', - mock.MagicMock(side_effect=Exception('Mock query exception.'))): - tracker._bootstrap() - eventlet.sleep(1) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - tracker.shutdown() - - # Ensure deletes are called. - self.assertEqual(2, querier._delete_state_object.call_count) - - def test_keep_state_object_on_error_at_query(self): - tracker = self._get_tracker() - querier = tracker.get_querier('test_querymodule') - querier._delete_state_object = mock.Mock(return_value=None) - querier.delete_state_object_on_error = False - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - with mock.patch.object( - querier.__class__, 'query', - mock.MagicMock(side_effect=Exception('Mock query exception.'))): - tracker._bootstrap() - eventlet.sleep(1) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - tracker.shutdown() - - # Ensure deletes are not called. - querier._delete_state_object.assert_not_called() - - def test_delete_state_object_on_error_at_update_result(self): - tracker = self._get_tracker() - querier = tracker.get_querier('test_querymodule') - querier._delete_state_object = mock.Mock(return_value=None) - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - with mock.patch.object( - querier.__class__, '_update_action_results', - mock.MagicMock(side_effect=Exception('Mock update exception.'))): - tracker._bootstrap() - eventlet.sleep(1) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - tracker.shutdown() - - # Ensure deletes are called. - self.assertEqual(2, querier._delete_state_object.call_count) - - def test_keep_state_object_on_error_at_update_result(self): - tracker = self._get_tracker() - querier = tracker.get_querier('test_querymodule') - querier._delete_state_object = mock.Mock(return_value=None) - querier.delete_state_object_on_error = False - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - with mock.patch.object( - querier.__class__, '_update_action_results', - mock.MagicMock(side_effect=Exception('Mock update exception.'))): - tracker._bootstrap() - eventlet.sleep(1) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - tracker.shutdown() - - # Ensure deletes are not called. - querier._delete_state_object.assert_not_called() - - @mock.patch.object( - Action, 'get_by_ref', mock.MagicMock(return_value='foobar')) - def test_execution_cancellation(self): - tracker = self._get_tracker() - querier = tracker.get_querier('test_querymodule') - querier._delete_state_object = mock.Mock(return_value=None) - runners_utils.invoke_post_run = mock.Mock(return_value=None) - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - with mock.patch.object( - querier.__class__, 'query', - mock.MagicMock(return_value=(action_constants.LIVEACTION_STATUS_CANCELED, {}))): - tracker._bootstrap() - eventlet.sleep(2) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertDictEqual(exec_db.result, {}) - - tracker.shutdown() - - # Ensure deletes are called. - self.assertEqual(2, querier._delete_state_object.call_count) - - # Ensure invoke_post_run is called. - self.assertEqual(2, runners_utils.invoke_post_run.call_count) - - @mock.patch.object( - Action, 'get_by_ref', mock.MagicMock(return_value=None)) - def test_action_deleted(self): - tracker = self._get_tracker() - action_db_utils.get_runnertype_by_name = mock.Mock(return_value=None) - - # Ensure state objects are present. - state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id) - state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id) - self.assertIsNotNone(state1) - self.assertIsNotNone(state2) - - # Process the state objects. - tracker._bootstrap() - eventlet.sleep(1) - - exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertIsNotNone(exec_db.result['called_with'][exec_id], exec_db.result) - - exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) - exec_db = LiveAction.get_by_id(exec_id) - self.assertIsNotNone(exec_db.result['called_with'][exec_id], exec_db.result) - - tracker.shutdown() - - # Ensure state objects are deleted. - self.assertRaises( - StackStormDBObjectNotFoundError, - ActionExecutionState.get_by_id, - ResultsTrackerTests.states['state1.yaml'].id - ) - - self.assertRaises( - StackStormDBObjectNotFoundError, - ActionExecutionState.get_by_id, - ResultsTrackerTests.states['state2.yaml'].id - ) - - # Ensure get_runnertype_by_name in invoke_post_run is not called. - action_db_utils.get_runnertype_by_name.assert_not_called() - - def _get_tracker(self): - from st2actions.resultstracker import resultstracker - tracker = resultstracker.get_tracker() - return tracker - - @classmethod - def _update_state_models(cls): - states = ResultsTrackerTests.states - state1 = ActionExecutionState.get_by_id(states['state1.yaml'].id) - state1.execution_id = ResultsTrackerTests.liveactions['liveaction1.yaml'].id - state2 = ActionExecutionState.get_by_id(states['state2.yaml'].id) - state2.execution_id = ResultsTrackerTests.liveactions['liveaction2.yaml'].id - ResultsTrackerTests.states['state1.yaml'] = ActionExecutionState.add_or_update(state1) - ResultsTrackerTests.states['state2.yaml'] = ActionExecutionState.add_or_update(state2) diff --git a/st2api/tests/unit/controllers/v1/test_packs.py b/st2api/tests/unit/controllers/v1/test_packs.py index 2fc64b1332..9406a50af0 100644 --- a/st2api/tests/unit/controllers/v1/test_packs.py +++ b/st2api/tests/unit/controllers/v1/test_packs.py @@ -540,15 +540,16 @@ def test_packs_register_endpoint(self, mock_get_packs): {'packs': ['dummy_pack_1'], 'types': ['action']}) self.assertEqual(resp.status_int, 200) - # 14 real plus 1 mock runner - self.assertEqual(resp.json, {'actions': 1, 'runners': 15}) + # 13 real plus 1 mock runner + self.assertEqual(resp.json, {'actions': 1, 'runners': 14}) # Verify that plural name form also works resp = self.app.post_json('/v1/packs/register', {'packs': ['dummy_pack_1'], 'types': ['actions']}) self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.json, {'actions': 1, 'runners': 15}) + # 13 real plus 1 mock runner + self.assertEqual(resp.json, {'actions': 1, 'runners': 14}) # Register single resource from a single pack specified multiple times - verify that # resources from the same pack are only registered once @@ -558,7 +559,8 @@ def test_packs_register_endpoint(self, mock_get_packs): 'fail_on_failure': False}) self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.json, {'actions': 1, 'runners': 15}) + # 13 real plus 1 mock runner + self.assertEqual(resp.json, {'actions': 1, 'runners': 14}) # Register resources from a single (non-existent pack) resp = self.app.post_json('/v1/packs/register', {'packs': ['doesntexist']}, diff --git a/st2common/bin/st2ctl b/st2common/bin/st2ctl index 29c71aae85..f2cd03ee22 100755 --- a/st2common/bin/st2ctl +++ b/st2common/bin/st2ctl @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -COMPONENTS="st2actionrunner st2api st2stream st2auth st2garbagecollector st2notifier st2resultstracker st2rulesengine st2sensorcontainer st2chatops st2timersengine st2workflowengine st2scheduler" +COMPONENTS="st2actionrunner st2api st2stream st2auth st2garbagecollector st2notifier st2rulesengine st2sensorcontainer st2chatops st2timersengine st2workflowengine st2scheduler" ST2_CONF="/etc/st2/st2.conf" # Ensure global environment is sourced if exists diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index 9200b9a4f6..8ad77fa626 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -462,25 +462,6 @@ def register_opts(ignore_errors=False): do_register_opts(coord_opts, 'coordination', ignore_errors) - # Results Tracker query module options - # Note that these are currently not used - query_opts = [ - cfg.IntOpt( - 'thread_pool_size', default=10, - help='Number of threads to use to query external workflow systems.'), - cfg.FloatOpt( - 'query_interval', default=5, - help='Time interval between queries to external workflow system.'), - cfg.FloatOpt( - 'empty_q_sleep_time', default=1, - help='Sleep delay in between queries when query queue is empty.'), - cfg.FloatOpt( - 'no_workers_sleep_time', default=1, - help='Sleep delay for query when there is no more worker in pool.') - ] - - do_register_opts(query_opts, group='resultstracker', ignore_errors=ignore_errors) - # XXX: This is required for us to support deprecated config group results_tracker query_opts = [ cfg.IntOpt( diff --git a/st2common/st2common/constants/runners.py b/st2common/st2common/constants/runners.py index 12fdc617f1..fe78a6497f 100644 --- a/st2common/st2common/constants/runners.py +++ b/st2common/st2common/constants/runners.py @@ -87,5 +87,3 @@ # Namespaces for dynamically loaded runner modules RUNNERS_NAMESPACE = "st2common.runners.runner" -RUNNERS_QUERY_MODULES_NAMESPACE = 'st2common.runners.query' -RUNNERS_CALLBACK_MODULES_NAMESPACE = 'st2common.runners.callback' diff --git a/st2common/st2common/query/__init__.py b/st2common/st2common/query/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2common/st2common/query/base.py b/st2common/st2common/query/base.py deleted file mode 100644 index 83c03e55d0..0000000000 --- a/st2common/st2common/query/base.py +++ /dev/null @@ -1,243 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -import abc -import eventlet -import six.moves.queue -import six -import time - -from oslo_config import cfg - -from st2common import log as logging -from st2common.constants import action as action_constants -from st2common.exceptions import db as db_exc -from st2common.persistence.executionstate import ActionExecutionState -from st2common.persistence.liveaction import LiveAction -from st2common.runners import utils as runners_utils -from st2common.services import executions -from st2common.util import date as date_utils - -LOG = logging.getLogger(__name__) - -__all__ = [ - 'Querier', - 'QueryContext' -] - - -@six.add_metaclass(abc.ABCMeta) -class Querier(object): - delete_state_object_on_error = True - - def _get_config_value(self, config_option): - config_value = None - - if 'results_tracker' in cfg.CONF and config_option in cfg.CONF.results_tracker: - config_value = getattr(cfg.CONF.results_tracker, config_option) - LOG.warning('You are using deprecated config group "results_tracker" for "%s". ' - 'Please use "resultstracker" group instead.', config_option) - - if not config_value and config_option in cfg.CONF.resultstracker: - config_value = getattr(cfg.CONF.resultstracker, config_option) - - return config_value - - def __init__(self): - self._empty_q_sleep_time = self._get_config_value('empty_q_sleep_time') - self._no_workers_sleep_time = self._get_config_value('no_workers_sleep_time') - self._query_interval = self._get_config_value('query_interval') - self._query_thread_pool_size = self._get_config_value('thread_pool_size') - self._query_contexts = six.moves.queue.Queue() - self._thread_pool = eventlet.GreenPool(self._query_thread_pool_size) - self._started = False - - def start(self): - self._started = True - while True: - while self._query_contexts.empty(): - eventlet.greenthread.sleep(self._empty_q_sleep_time) - while self._thread_pool.free() <= 0: - eventlet.greenthread.sleep(self._no_workers_sleep_time) - self._fire_queries() - eventlet.sleep(self._query_interval) - - def add_queries(self, query_contexts=None): - if query_contexts is None: - query_contexts = [] - LOG.debug('Adding queries to querier: %s' % query_contexts) - for query_context in query_contexts: - self._query_contexts.put((time.time(), query_context)) - - def is_started(self): - return self._started - - def _fire_queries(self, blocking=False): - if self._thread_pool.free() <= 0: - return - - now = time.time() - reschedule_queries = [] - - while not self._query_contexts.empty() and self._thread_pool.free() > 0: - (last_query_time, query_context) = self._query_contexts.get_nowait() - if now - last_query_time < self._query_interval: - reschedule_queries.append((last_query_time, query_context)) - continue - else: - if not blocking: - self._thread_pool.spawn( - self._query_and_save_results, - query_context, - last_query_time - ) - # Add an option to block and execute the function directly for unit tests. - else: - self._query_and_save_results( - query_context, - last_query_time - ) - - for query in reschedule_queries: - self._query_contexts.put((query[0], query[1])) - - def _query_and_save_results(self, query_context, last_query_time=None): - this_query_time = time.time() - execution_id = query_context.execution_id - actual_query_context = query_context.query_context - - LOG.debug('Querying external service for results. Context: %s' % actual_query_context) - try: - (status, results) = self.query( - execution_id, - actual_query_context, - last_query_time=last_query_time - ) - except: - LOG.exception('Failed querying results for liveaction_id %s.', execution_id) - if self.delete_state_object_on_error: - self._delete_state_object(query_context) - LOG.debug('Removed state object %s.', query_context) - return - - liveaction_db = None - try: - liveaction_db = self._update_action_results(execution_id, status, results) - except Exception: - LOG.exception('Failed updating action results for liveaction_id %s', execution_id) - if self.delete_state_object_on_error: - self._delete_state_object(query_context) - LOG.debug('Removed state object %s.', query_context) - return - - if (status in action_constants.LIVEACTION_COMPLETED_STATES or - status == action_constants.LIVEACTION_STATUS_PAUSED): - runners_utils.invoke_post_run(liveaction_db) - self._delete_state_object(query_context) - LOG.debug( - "Detailed workflow liveaction results - ", extra={'liveaction_db': liveaction_db} - ) - return - - if not self._is_state_object_exist(query_context): - LOG.warning( - 'Query for liveaction_id %s is not rescheduled ' - 'because state object %s has been deleted.', - execution_id, - query_context.id - ) - - return - - self._query_contexts.put((this_query_time, query_context)) - - def _update_action_results(self, execution_id, status, results): - liveaction_db = LiveAction.get_by_id(execution_id) - if not liveaction_db: - raise Exception('No DB model for liveaction_id: %s' % execution_id) - - if liveaction_db.status != action_constants.LIVEACTION_STATUS_CANCELED: - liveaction_db.status = status - - liveaction_db.result = results - - # Action has completed, record end_timestamp - if (liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES and - not liveaction_db.end_timestamp): - liveaction_db.end_timestamp = date_utils.get_datetime_utc_now() - - # update liveaction, update actionexecution and then publish update. - updated_liveaction = LiveAction.add_or_update(liveaction_db, publish=False) - executions.update_execution(updated_liveaction) - LiveAction.publish_update(updated_liveaction) - - return updated_liveaction - - def _delete_state_object(self, query_context): - state_db = None - - try: - state_db = ActionExecutionState.get_by_id(query_context.id) - except db_exc.StackStormDBObjectNotFoundError: - pass - - if state_db is not None: - try: - LOG.info('Clearing state object: %s', state_db) - ActionExecutionState.delete(state_db) - except: - LOG.exception('Failed clearing state object: %s', state_db) - - def _is_state_object_exist(self, query_context): - state_db = None - - try: - state_db = ActionExecutionState.get_by_id(query_context.id) - except db_exc.StackStormDBObjectNotFoundError: - pass - - return (state_db is not None) - - def query(self, execution_id, query_context, last_query_time=None): - """ - This is the method individual queriers must implement. - This method should return a tuple of (status, results). - - status should be one of LIVEACTION_STATUS_SUCCEEDED, LIVEACTION_STATUS_RUNNING, - LIVEACTION_STATUS_FAILED defined in st2common.constants.action. - """ - raise NotImplementedError() - - def print_stats(self): - LOG.info('\t --- Name: %s, pending queuries: %d', self.__class__.__name__, - self._query_contexts.qsize()) - - -class QueryContext(object): - def __init__(self, obj_id, execution_id, query_context, query_module): - self.id = obj_id - self.execution_id = execution_id - self.query_context = query_context - self.query_module = query_module - - @classmethod - def from_model(cls, model): - return QueryContext(str(model.id), str(model.execution_id), model.query_context, - model.query_module) - - def __repr__(self): - return ('' % - (self.id, self.execution_id, self.query_context)) diff --git a/st2common/st2common/runners/base.py b/st2common/st2common/runners/base.py index 6d7a1ade51..6a9656b9a1 100644 --- a/st2common/st2common/runners/base.py +++ b/st2common/st2common/runners/base.py @@ -29,8 +29,6 @@ from st2common.constants import action as action_constants from st2common.constants import pack as pack_constants from st2common.constants.runners import RUNNERS_NAMESPACE -from st2common.constants.runners import RUNNERS_QUERY_MODULES_NAMESPACE -from st2common.constants.runners import RUNNERS_CALLBACK_MODULES_NAMESPACE from st2common.content.utils import get_pack_directory from st2common.content.utils import get_pack_base_path from st2common.exceptions import actionrunner as exc @@ -55,9 +53,6 @@ 'get_runner', 'get_metadata', - - 'get_query_module', - 'get_callback_module' ] @@ -117,38 +112,6 @@ def get_runner_module(name): return module -def get_query_module(name): - """ - Retrieve runner query module for the provided runner. - """ - # NOTE: For backward compatibility we also support "_" in place of "-" - from stevedore.exception import NoMatches - - try: - module = get_plugin_instance(RUNNERS_QUERY_MODULES_NAMESPACE, name, invoke_on_load=False) - except NoMatches: - name = name.replace('_', '-') - module = get_plugin_instance(RUNNERS_QUERY_MODULES_NAMESPACE, name, invoke_on_load=False) - - return module - - -def get_callback_module(name): - """ - Retrieve runner callback module for the provided runner. - """ - # NOTE: For backward compatibility we also support "_" in place of "-" - from stevedore.exception import NoMatches - - try: - module = get_plugin_instance(RUNNERS_CALLBACK_MODULES_NAMESPACE, name, invoke_on_load=False) - except NoMatches: - name = name.replace('_', '-') - module = get_plugin_instance(RUNNERS_CALLBACK_MODULES_NAMESPACE, name, invoke_on_load=False) - - return module - - def get_metadata(package_name): """ Return runner related metadata for the provided runner package name. @@ -190,7 +153,6 @@ def __init__(self, runner_id): self.entry_point = None self.libs_dir_path = None self.context = None - self.callback = None self.auth_token = None self.rerun_ex_ref = None @@ -228,10 +190,7 @@ def cancel(self): ) def post_run(self, status, result): - if self.callback and isinstance(self.callback, dict) and 'source' in self.callback: - callback_module = get_callback_module(self.callback['source']) - callback_handler = callback_module.get_instance() - callback_handler.callback(self.liveaction) + pass @deprecated def get_pack_name(self): diff --git a/st2common/st2common/util/loader.py b/st2common/st2common/util/loader.py index 3acdd1e090..0e27a0da32 100644 --- a/st2common/st2common/util/loader.py +++ b/st2common/st2common/util/loader.py @@ -44,8 +44,6 @@ # Cache for dynamically loaded runner modules RUNNER_MODULES_CACHE = defaultdict(dict) -QUERIER_MODULES_CACHE = {} -CALLBACK_MODULES_CACHE = {} def _register_plugin_path(plugin_dir_abs_path): diff --git a/st2common/tests/runners/mock_query_callback/MANIFEST.in b/st2common/tests/runners/mock_query_callback/MANIFEST.in deleted file mode 100644 index 25ce80c091..0000000000 --- a/st2common/tests/runners/mock_query_callback/MANIFEST.in +++ /dev/null @@ -1,11 +0,0 @@ -# https://docs.python.org/2/distutils/sourcedist.html#commands -# Include all files under the source tree by default. -# Another behaviour can be used in the future though. -include __init__.py -include runner.yaml -include dist_utils.py -include requirements.txt -include README.rst -include CHANGELOG.rst -include LICENSE -global-exclude *.pyc diff --git a/st2common/tests/runners/mock_query_callback/callback/__init__.py b/st2common/tests/runners/mock_query_callback/callback/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2common/tests/runners/mock_query_callback/dist_utils.py b/st2common/tests/runners/mock_query_callback/dist_utils.py deleted file mode 100644 index 1d637ca664..0000000000 --- a/st2common/tests/runners/mock_query_callback/dist_utils.py +++ /dev/null @@ -1,169 +0,0 @@ -# -*- coding: utf-8 -*- -# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY -# Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to -# update dist_utils.py files for all components - -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -import os -import re -import sys - -from distutils.version import StrictVersion - -# NOTE: This script can't rely on any 3rd party dependency so we need to use this code here -# -# TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import -# from pip? -# -# TODO: Dear future developer, if you are back here fixing a bug with how we parse -# requirements files, please look into using the packaging package on PyPI: -# https://packaging.pypa.io/en/latest/requirements/ -# and specifying that in the `setup_requires` argument to `setuptools.setup()` -# for subpackages. -# At the very least we can vendorize some of their code instead of reimplementing -# each piece of their code every time our parsing breaks. -PY3 = sys.version_info[0] == 3 - -if PY3: - text_type = str -else: - text_type = unicode # NOQA - -GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' - -__all__ = [ - 'check_pip_is_installed', - 'check_pip_version', - 'fetch_requirements', - 'apply_vagrant_workaround', - 'get_version_string', - 'parse_version_string' -] - - -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print('Failed to import pip: %s' % (text_type(e))) - print('') - print('Download pip:\n%s' % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version='6.0.0'): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print("Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, - min_version, - GET_PIP)) - sys.exit(1) - - return True - - -def fetch_requirements(requirements_file_path): - """ - Return a list of requirements and links by parsing the provided requirements file. - """ - links = [] - reqs = [] - - def _get_link(line): - vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] - - for vcs_prefix in vcs_prefixes: - if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): - req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) - - if not req_name: - req_name = re.findall('.*#egg=(.+?)$', line) - else: - req_name = req_name[0] - - if not req_name: - raise ValueError('Line "%s" is missing "#egg="' % (line)) - - link = line.replace('-e ', '').strip() - return link, req_name[0] - - return None, None - - with open(requirements_file_path, 'r') as fp: - for line in fp.readlines(): - line = line.strip() - - if line.startswith('#') or not line: - continue - - link, req_name = _get_link(line=line) - - if link: - links.append(link) - else: - req_name = line - - if ';' in req_name: - req_name = req_name.split(';')[0].strip() - - reqs.append(req_name) - - return (reqs, links) - - -def apply_vagrant_workaround(): - """ - Function which detects if the script is being executed inside vagrant and if it is, it deletes - "os.link" attribute. - Note: Without this workaround, setup.py sdist will fail when running inside a shared directory - (nfs / virtualbox shared folders). - """ - if os.environ.get('USER', None) == 'vagrant': - del os.link - - -def get_version_string(init_file): - """ - Read __version__ string for an init file. - """ - - with open(init_file, 'r') as fp: - content = fp.read() - version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", - content, re.M) - if version_match: - return version_match.group(1) - - raise RuntimeError('Unable to find version string in %s.' % (init_file)) - - -# alias for get_version_string -parse_version_string = get_version_string diff --git a/st2common/tests/runners/mock_query_callback/in-requirements.txt b/st2common/tests/runners/mock_query_callback/in-requirements.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2common/tests/runners/mock_query_callback/mock_query_callback/__init__.py b/st2common/tests/runners/mock_query_callback/mock_query_callback/__init__.py deleted file mode 100644 index e682f5e7b6..0000000000 --- a/st2common/tests/runners/mock_query_callback/mock_query_callback/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__version__ = '3.3dev' diff --git a/st2common/tests/runners/mock_query_callback/mock_query_callback/callback.py b/st2common/tests/runners/mock_query_callback/mock_query_callback/callback.py deleted file mode 100644 index 9f41758bab..0000000000 --- a/st2common/tests/runners/mock_query_callback/mock_query_callback/callback.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -from st2common.callback import base as callback - - -def get_instance(): - return MockCallbackHandler - - -class MockCallbackHandler(callback.AsyncActionExecutionCallbackHandler): - - @classmethod - def callback(cls, liveaction): - pass diff --git a/st2common/tests/runners/mock_query_callback/mock_query_callback/mock_query_callback.py b/st2common/tests/runners/mock_query_callback/mock_query_callback/mock_query_callback.py deleted file mode 100644 index 7c78fcc134..0000000000 --- a/st2common/tests/runners/mock_query_callback/mock_query_callback/mock_query_callback.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED -from st2common.runners.base import ActionRunner -from st2common.runners.base import get_metadata as get_runner_metadata - -import uuid - - -__all__ = [ - 'MockQueryCallbackRunner', - - 'get_runner', - 'get_metadata' -] - - -class MockQueryCallbackRunner(ActionRunner): - """ - Runner which does absolutely nothing. No-op action. - """ - - def __init__(self, runner_id): - super(MockQueryCallbackRunner, self).__init__(runner_id=runner_id) - - def pre_run(self): - super(MockQueryCallbackRunner, self).pre_run() - - def run(self, action_parameters): - result = { - 'failed': False, - 'succeeded': True, - 'return_code': 0, - } - - status = LIVEACTION_STATUS_SUCCEEDED - return (status, result, None) - - -def get_runner(): - return MockQueryCallbackRunner(str(uuid.uuid4())) - - -def get_metadata(): - return get_runner_metadata('noop_runner')[0] diff --git a/st2common/tests/runners/mock_query_callback/mock_query_callback/query.py b/st2common/tests/runners/mock_query_callback/mock_query_callback/query.py deleted file mode 100644 index 065dd5d0e6..0000000000 --- a/st2common/tests/runners/mock_query_callback/mock_query_callback/query.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -from st2common.query.base import Querier -from st2common.constants import action as action_constants - - -def get_instance(): - return MockQueryCallbackQuerier() - - -class MockQueryCallbackQuerier(Querier): - - def __init__(self, *args, **kwargs): - super(MockQueryCallbackQuerier, self).__init__(*args, **kwargs) - - def query(self, execution_id, query_context, last_query_time=None): - return (action_constants.LIVEACTION_STATUS_SUCCEEDED, - {'called_with': {execution_id: query_context}}) diff --git a/st2common/tests/runners/mock_query_callback/mock_query_callback/runner.yaml b/st2common/tests/runners/mock_query_callback/mock_query_callback/runner.yaml deleted file mode 100644 index 318f0f555b..0000000000 --- a/st2common/tests/runners/mock_query_callback/mock_query_callback/runner.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- aliases: [] - description: A mock runner for query and callback - enabled: true - name: mock_query_callback - query_module: mock_query_callback - runner_module: mock_query_callback - diff --git a/st2common/tests/runners/mock_query_callback/query/__init__.py b/st2common/tests/runners/mock_query_callback/query/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2common/tests/runners/mock_query_callback/requirements.txt b/st2common/tests/runners/mock_query_callback/requirements.txt deleted file mode 100644 index f4a1c5a5ee..0000000000 --- a/st2common/tests/runners/mock_query_callback/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Don't edit this file. It's generated automatically! -# If you want to update global dependencies, modify fixed-requirements.txt -# and then run 'make requirements' to update requirements.txt for all -# components. -# If you want to update depdencies for a single component, modify the -# in-requirements.txt for that component and then run 'make requirements' to -# update the component requirements.txt - diff --git a/st2common/tests/runners/mock_query_callback/runner.yaml b/st2common/tests/runners/mock_query_callback/runner.yaml deleted file mode 120000 index 994fdf0c6b..0000000000 --- a/st2common/tests/runners/mock_query_callback/runner.yaml +++ /dev/null @@ -1 +0,0 @@ -mock_query_callback/runner.yaml \ No newline at end of file diff --git a/st2common/tests/runners/mock_query_callback/setup.py b/st2common/tests/runners/mock_query_callback/setup.py deleted file mode 100644 index af00f5b9cc..0000000000 --- a/st2common/tests/runners/mock_query_callback/setup.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -import os.path - -from setuptools import setup -from setuptools import find_packages - -from dist_utils import fetch_requirements -from dist_utils import apply_vagrant_workaround - -from mock_query_callback import __version__ - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -REQUIREMENTS_FILE = os.path.join(BASE_DIR, 'requirements.txt') - -install_reqs, dep_links = fetch_requirements(REQUIREMENTS_FILE) - -apply_vagrant_workaround() -setup( - name='stackstorm-runner-mock_query_callback', - version=__version__, - description=('Mock runner for query callback'), - author='StackStorm', - author_email='info@stackstorm.com', - license='Apache License (2.0)', - url='https://stackstorm.com/', - install_requires=install_reqs, - dependency_links=dep_links, - test_suite='tests', - zip_safe=False, - include_package_data=True, - packages=find_packages(exclude=['setuptools', 'tests']), - package_data={'mock_query_callback': ['runner.yaml']}, - scripts=[], - entry_points={ - 'st2common.runners.runner': [ - 'mock_query_callback = mock_query_callback.mock_query_callback', - ], - 'st2common.runners.query': [ - 'mock_query_callback = mock_query_callback.query', - ], - 'st2common.runners.callback': [ - 'mock_query_callback = mock_query_callback.callback', - ], - } -) diff --git a/st2common/tests/unit/test_query_base.py b/st2common/tests/unit/test_query_base.py deleted file mode 100644 index a1edf0610a..0000000000 --- a/st2common/tests/unit/test_query_base.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -import six.moves.queue -import time -import uuid - -from unittest2 import TestCase -import mock - -from st2common.constants import action as action_constants -from st2common.query.base import Querier, QueryContext -from st2common.runners import utils as runners_utils -from st2tests.config import parse_args -parse_args() - - -class QueryBaseTests(TestCase): - - @mock.patch.object( - Querier, - '_query_and_save_results', - mock.MagicMock(return_value=True) - ) - def test_fire_queries_doesnt_loop(self): - querier = Querier() - - mock_query_state_1 = QueryContext( - uuid.uuid4().hex, - uuid.uuid4().hex, - 'dummy_runner', - { - 'dummy': { - 'workflow_name': 'st2ci.st2_pkg_e2e_test', - 'execution_id': '6d624534-42ca-425c-aa3a-ccc676386fb2' - } - } - ) - - mock_query_state_2 = QueryContext( - uuid.uuid4().hex, - uuid.uuid4().hex, - 'dummy_runner', - { - 'dummy': { - 'workflow_name': 'st2ci.st2_pkg_e2e_test', - 'execution_id': '6d624534-42ca-425c-aa3a-ccc676386fb3' - } - } - ) - - mock_query_state_3 = QueryContext( - uuid.uuid4().hex, - uuid.uuid4().hex, - 'dummy_runner', - { - 'dummy': { - 'workflow_name': 'st2ci.st2_pkg_e2e_test', - 'execution_id': '6d624534-42ca-425c-aa3a-ccc676386fb4' - } - } - ) - - now = time.time() - query_contexts = six.moves.queue.Queue() - query_contexts.put((now + 100000, mock_query_state_1)), - query_contexts.put((now + 100001, mock_query_state_2)), - query_contexts.put((now - 200000, mock_query_state_3)), - querier._query_contexts = query_contexts - querier._fire_queries() - self.assertEqual(querier._query_contexts.qsize(), 2) - - @mock.patch.object( - Querier, - 'query', - mock.MagicMock(return_value=(action_constants.LIVEACTION_STATUS_RUNNING, {})) - ) - @mock.patch.object( - Querier, - '_update_action_results', - mock.MagicMock(return_value=None) - ) - @mock.patch.object( - Querier, - '_is_state_object_exist', - mock.MagicMock(return_value=True) - ) - @mock.patch.object( - Querier, - '_delete_state_object', - mock.MagicMock(return_value=None) - ) - def test_query_rescheduled(self): - querier = Querier() - - mock_query_state_1 = QueryContext( - uuid.uuid4().hex, - uuid.uuid4().hex, - 'dummy_runner', - { - 'dummy': { - 'workflow_name': 'st2ci.st2_pkg_e2e_test', - 'execution_id': '6d624534-42ca-425c-aa3a-ccc676386fb2' - } - } - ) - - now = time.time() - query_contexts = six.moves.queue.Queue() - query_contexts.put((now - 200000, mock_query_state_1)), - querier._query_contexts = query_contexts - querier._fire_queries(blocking=True) - self.assertFalse(Querier._delete_state_object.called) - self.assertEqual(querier._query_contexts.qsize(), 1) - - @mock.patch.object( - Querier, - 'query', - mock.MagicMock(return_value=(action_constants.LIVEACTION_STATUS_SUCCEEDED, {})) - ) - @mock.patch.object( - Querier, - '_update_action_results', - mock.MagicMock(return_value=None) - ) - @mock.patch.object( - Querier, - '_is_state_object_exist', - mock.MagicMock(return_value=True) - ) - @mock.patch.object( - Querier, - '_delete_state_object', - mock.MagicMock(return_value=None) - ) - @mock.patch.object( - runners_utils, - 'invoke_post_run', - mock.MagicMock(return_value=None) - ) - def test_query_completed(self): - querier = Querier() - - mock_query_state_1 = QueryContext( - uuid.uuid4().hex, - uuid.uuid4().hex, - 'dummy_runner', - { - 'dummy': { - 'workflow_name': 'st2ci.st2_pkg_e2e_test', - 'execution_id': '6d624534-42ca-425c-aa3a-ccc676386fb2' - } - } - ) - - now = time.time() - query_contexts = six.moves.queue.Queue() - query_contexts.put((now - 200000, mock_query_state_1)), - querier._query_contexts = query_contexts - querier._fire_queries(blocking=True) - self.assertTrue(runners_utils.invoke_post_run.called) - self.assertTrue(Querier._delete_state_object.called) - self.assertEqual(querier._query_contexts.qsize(), 0) - - @mock.patch.object( - Querier, - 'query', - mock.MagicMock(return_value=(action_constants.LIVEACTION_STATUS_RUNNING, {})) - ) - @mock.patch.object( - Querier, - '_update_action_results', - mock.MagicMock(return_value=None) - ) - @mock.patch.object( - Querier, - '_is_state_object_exist', - mock.MagicMock(return_value=False) - ) - @mock.patch.object( - Querier, - '_delete_state_object', - mock.MagicMock(return_value=None) - ) - def test_state_db_entry_deleted(self): - querier = Querier() - - mock_query_state_1 = QueryContext( - uuid.uuid4().hex, - uuid.uuid4().hex, - 'dummy_runner', - { - 'dummy': { - 'workflow_name': 'st2ci.st2_pkg_e2e_test', - 'execution_id': '6d624534-42ca-425c-aa3a-ccc676386fb2' - } - } - ) - - now = time.time() - query_contexts = six.moves.queue.Queue() - query_contexts.put((now - 200000, mock_query_state_1)), - querier._query_contexts = query_contexts - querier._fire_queries(blocking=True) - self.assertFalse(Querier._delete_state_object.called) - self.assertEqual(querier._query_contexts.qsize(), 0) diff --git a/st2common/tests/unit/test_runners_base.py b/st2common/tests/unit/test_runners_base.py index 54d13ee18d..34ede41adf 100644 --- a/st2common/tests/unit/test_runners_base.py +++ b/st2common/tests/unit/test_runners_base.py @@ -15,11 +15,7 @@ from __future__ import absolute_import -from stevedore.exception import NoMatches - from st2common.runners.base import get_runner -from st2common.runners.base import get_query_module -from st2common.runners.base import get_callback_module from st2common.exceptions.actionrunner import ActionRunnerCreateError from st2tests.base import DbTestCase @@ -35,25 +31,3 @@ def test_get_runner_failure_not_found(self): expected_msg = 'Failed to find runner invalid-name-not-found.*' self.assertRaisesRegexp(ActionRunnerCreateError, expected_msg, get_runner, 'invalid-name-not-found') - - def test_get_query_module_success(self): - query_module = get_query_module('mock_query_callback') - - self.assertEqual(query_module.__name__, 'mock_query_callback.query') - self.assertTrue(query_module.get_instance()) - - def test_get_query_module_failure_not_found(self): - expected_msg = 'No .*? driver found.*' - self.assertRaisesRegexp(NoMatches, expected_msg, - get_query_module, 'invalid-name-not-found') - - def test_get_callback_module_success(self): - callback_module = get_callback_module('mock_query_callback') - - self.assertEqual(callback_module.__name__, 'mock_query_callback.callback') - self.assertTrue(callback_module.get_instance()) - - def test_get_callback_module_failure_not_found(self): - expected_msg = 'No .*? driver found.*' - self.assertRaisesRegexp(NoMatches, expected_msg, - get_callback_module, 'invalid-name-not-found') diff --git a/st2tests/conf/st2.conf b/st2tests/conf/st2.conf index 9dadee986c..0801ccc4a7 100644 --- a/st2tests/conf/st2.conf +++ b/st2tests/conf/st2.conf @@ -50,6 +50,3 @@ remote_dir = /tmp [history] logging = st2actions/conf/logging.history.conf - -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py index 77dca923ab..7fa4ad7b6e 100644 --- a/st2tests/st2tests/config.py +++ b/st2tests/st2tests/config.py @@ -89,7 +89,6 @@ def _override_common_opts(): CONF.set_override(name='packs_base_paths', override=packs_base_path, group='content') CONF.set_override(name='api_url', override='http://127.0.0.1', group='auth') CONF.set_override(name='mask_secrets', override=True, group='log') - CONF.set_override(name='query_interval', override=0.1, group='resultstracker') CONF.set_override(name='stream_output', override=False, group='actionrunner') diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.audit_log_level.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.audit_log_level.conf index feb8afd095..a07773bc48 100644 --- a/st2tests/st2tests/fixtures/conf/st2.tests.api.audit_log_level.conf +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.audit_log_level.conf @@ -85,10 +85,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.debug_log_level.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.debug_log_level.conf index 06e5eb0561..0b435df722 100644 --- a/st2tests/st2tests/fixtures/conf/st2.tests.api.debug_log_level.conf +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.debug_log_level.conf @@ -85,10 +85,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.info_log_level.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.info_log_level.conf index dd72dfd1b6..45bf5f364c 100644 --- a/st2tests/st2tests/fixtures/conf/st2.tests.api.info_log_level.conf +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.info_log_level.conf @@ -85,10 +85,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true.conf index 7fa1f26e40..ee8b2c9e07 100644 --- a/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true.conf +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true.conf @@ -85,10 +85,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true_logging_debug.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true_logging_debug.conf index 8172c6650a..caaf4bbfba 100644 --- a/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true_logging_debug.conf +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true_logging_debug.conf @@ -85,10 +85,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.conf b/st2tests/st2tests/fixtures/conf/st2.tests.conf index 694cfb082c..da7386382a 100644 --- a/st2tests/st2tests/fixtures/conf/st2.tests.conf +++ b/st2tests/st2tests/fixtures/conf/st2.tests.conf @@ -86,10 +86,6 @@ url = amqp://guest:guest@127.0.0.1:5672/ [ssh_runner] remote_dir = /tmp -[resultstracker] -logging = st2actions/conf/logging.resultstracker.conf -query_interval = 0.1 - [notifier] logging = st2actions/conf/logging.notifier.conf diff --git a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/__init__.py b/st2tests/st2tests/fixtures/packs/runners/test_querymodule/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/callback/__init__.py b/st2tests/st2tests/fixtures/packs/runners/test_querymodule/callback/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/callback/test_querymodule.py b/st2tests/st2tests/fixtures/packs/runners/test_querymodule/callback/test_querymodule.py deleted file mode 100644 index 8d9e0847b7..0000000000 --- a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/callback/test_querymodule.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from st2common import log as logging -from st2common.callback import base as callback - - -LOG = logging.getLogger(__name__) - - -def get_instance(): - return MockRunnerCallbackHandler - - -class MockRunnerCallbackHandler(callback.AsyncActionExecutionCallbackHandler): - - @classmethod - def callback(cls, url, context, status, result): - pass diff --git a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/query/__init__.py b/st2tests/st2tests/fixtures/packs/runners/test_querymodule/query/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/query/test_querymodule.py b/st2tests/st2tests/fixtures/packs/runners/test_querymodule/query/test_querymodule.py deleted file mode 100644 index 9dc65625df..0000000000 --- a/st2tests/st2tests/fixtures/packs/runners/test_querymodule/query/test_querymodule.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from st2common.query.base import Querier -from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED - - -class TestQuerier(Querier): - def __init__(self, *args, **kwargs): - super(TestQuerier, self).__init__(*args, **kwargs) - - def query(self, execution_id, query_context, last_query_time=None): - return (LIVEACTION_STATUS_SUCCEEDED, {'called_with': {execution_id: query_context}}) - - -def get_instance(): - return TestQuerier() diff --git a/tools/config_gen.py b/tools/config_gen.py index 06e2050ac0..e705161ea3 100755 --- a/tools/config_gen.py +++ b/tools/config_gen.py @@ -27,7 +27,6 @@ CONFIGS = ['st2actions.config', 'st2actions.scheduler.config', 'st2actions.notifier.config', - 'st2actions.resultstracker.config', 'st2actions.workflows.config', 'st2api.config', 'st2stream.config', diff --git a/tools/launchdev.sh b/tools/launchdev.sh index 20f63b4fb7..9e6fc9bc46 100755 --- a/tools/launchdev.sh +++ b/tools/launchdev.sh @@ -266,12 +266,6 @@ function st2start(){ ./st2reactor/bin/st2timersengine \ --config-file $ST2_CONF - # Run the results tracker - echo 'Starting screen session st2-resultstracker...' - screen -L -c tools/screen-configs/st2resultstracker.conf -d -m -S st2-resultstracker ${VIRTUALENV}/bin/python \ - ./st2actions/bin/st2resultstracker \ - --config-file $ST2_CONF - # Run the actions notifier echo 'Starting screen session st2-notifier...' screen -L -c tools/screen-configs/st2notifier.conf -d -m -S st2-notifier ${VIRTUALENV}/bin/python \ @@ -310,7 +304,6 @@ function st2start(){ "${RUNNER_SCREENS[@]}" "st2-sensorcontainer" "st2-rulesengine" - "st2-resultstracker" "st2-notifier" "st2-auth" "st2-timersengine" diff --git a/tools/screen-configs/st2resultstracker.conf b/tools/screen-configs/st2resultstracker.conf deleted file mode 100644 index c5d454d4d0..0000000000 --- a/tools/screen-configs/st2resultstracker.conf +++ /dev/null @@ -1,6 +0,0 @@ -logfile logs/screen-st2resultstracker.log -logfile flush 1 -log on -logtstamp after 1 -logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" -logtstamp on