diff --git a/CHANGELOG.rst b/CHANGELOG.rst index edce010f5a..b468e22a3f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,7 +12,10 @@ Added #4757 * Add ``user`` parameter to ``re_run`` method of st2client. #4785 * Install pack dependencies automatically. #4769 -* Add support for `immutable_parameters` on Action Aliases. This feature allows default parameters to be supplied to the action on every execution of the alias. #4786 +* Add support for `immutable_parameters` on Action Aliases. This feature allows default + parameters to be supplied to the action on every execution of the alias. #4786 +* Add ``get_entrypoint()`` method to ``ActionResourceManager`` attribute of st2client. + #4791 Changed ~~~~~~~ @@ -27,6 +30,15 @@ Changed writing very large executions (executions with large results) to the database. #4767 * Improved development instructions in requirements.txt and dist_utils.py comment headers (improvement) #4774 +* Add new ``actionrunner.stream_output_buffer_size`` config option and default it to ``-1`` + (previously default value was ``0``). This should result in a better performance and smaller + CPU utilization for Python runner actions which produce a lot of output. + (improvement) + + Reported and contributed by Joshua Meyer (@jdmeyer3) #4803 +* Add new ``action_runner.pip_opts`` st2.conf config option which allows user to specify a list + of command line option which are passed to ``pip install`` command when installing pack + dependencies into a pack specific virtual environment. #4792 Fixed ~~~~~ @@ -49,6 +61,21 @@ Fixed doesn't depend on internal pip API and so it works with latest pip version. (bug fix) #4750 * Fix dependency conflicts in pack CI runs: downgrade requests dependency back to 0.21.0, update internal dependencies and test expectations (amqp, pyyaml, prance, six) (bugfix) #4774 +* Fix secrets masking in action parameters section defined inside the rule when using + ``GET /v1/rules`` and ``GET /v1/rules/`` API endpoint. (bug fix) #4788 #4807 + + Contributed by @Nicodemos305 and @jeansfelix +* Fix a bug with authentication API endpoint (``POST /auth/v1/tokens``) returning internal + server error when running under gunicorn and when``auth.api_url`` config option was not set. + (bug fix) #4809 + + Reported by @guzzijones +* Fixed ``st2 execution get`` and ``st2 run`` not printing the ``action.ref`` for non-workflow + actions. (bug fix) #4739 + + Contributed by Nick Maludy (@nmaludy Encore Technologies) +* Update ``st2 execution get`` command to always include ``context.user``, ``start_timestamp`` and + ``end_timestamp`` attributes. (improvement) #4739 3.1.0 - June 27, 2019 --------------------- diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index d5d295d058..50e8042baf 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -8,6 +8,8 @@ enable = True emit_when = succeeded,failed,timeout,canceled,abandoned # comma separated list allowed here. [actionrunner] +# List of pip options to be passed to "pip install" command when installing pack dependencies into pack virtual environment. +pip_opts = # comma separated list allowed here. # Internal pool size for dispatcher used by regular actions. actions_pool_size = 60 # Default log level to use for Python runner actions. Can be overriden on invocation basis using "log_level" runner parameter. @@ -20,6 +22,8 @@ python3_prefix = None virtualenv_binary = /usr/bin/virtualenv # Python 3 binary which will be used by Python actions for packs which use Python 3 virtual environment. python3_binary = /usr/bin/python3 +# Buffer size to use for real time action output streaming. 0 means unbuffered 1 means line buffered, -1 means system default, which usually means fully buffered and any other positive value means use a buffer of (approximately) that size +stream_output_buffer_size = -1 # List of virtualenv options to be passsed to "virtualenv" command that creates pack virtualenv. virtualenv_opts = --system-site-packages # comma separated list allowed here. # True to store and stream action output (stdout and stderr) in real-time. diff --git a/contrib/runners/python_runner/python_runner/python_runner.py b/contrib/runners/python_runner/python_runner/python_runner.py index a1f42101b0..dae67f33e2 100644 --- a/contrib/runners/python_runner/python_runner/python_runner.py +++ b/contrib/runners/python_runner/python_runner/python_runner.py @@ -248,8 +248,11 @@ def run(self, action_parameters): if stdin_params: command_string = 'echo %s | %s' % (quote_unix(stdin_params), command_string) - LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' % (env['PATH'], env['PYTHONPATH'], - command_string)) + bufsize = cfg.CONF.actionrunner.stream_output_buffer_size + + LOG.debug('Running command (bufsize=%s): PATH=%s PYTHONPATH=%s %s' % (bufsize, env['PATH'], + env['PYTHONPATH'], + command_string)) exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdin=stdin, stdout=subprocess.PIPE, @@ -261,7 +264,8 @@ def run(self, action_parameters): read_stderr_func=read_and_store_stderr, read_stdout_buffer=stdout, read_stderr_buffer=stderr, - stdin_value=stdin_params) + stdin_value=stdin_params, + bufsize=bufsize) LOG.debug('Returning values: %s, %s, %s, %s', exit_code, stdout, stderr, timed_out) LOG.debug('Returning.') return self._get_output_values(exit_code, stdout, stderr, timed_out) diff --git a/contrib/runners/python_runner/tests/unit/test_pythonrunner.py b/contrib/runners/python_runner/tests/unit/test_pythonrunner.py index 9c242f237f..7dcad87d28 100644 --- a/contrib/runners/python_runner/tests/unit/test_pythonrunner.py +++ b/contrib/runners/python_runner/tests/unit/test_pythonrunner.py @@ -63,6 +63,8 @@ PRINT_CONFIG_ITEM_ACTION = os.path.join(tests_base.get_resources_path(), 'packs', 'pythonactions/actions/print_config_item_doesnt_exist.py') +PRINT_TO_STDOUT_STDERR_ACTION = os.path.join(tests_base.get_resources_path(), 'packs', + 'pythonactions/actions/print_to_stdout_and_stderr.py') # Note: runner inherits parent args which doesn't work with tests since test pass additional @@ -406,6 +408,31 @@ def test_action_stdout_and_stderr_is_stored_in_the_db(self, mock_spawn, mock_pop self.assertEqual(output_dbs[1].data, mock_stderr[1]) self.assertEqual(output_dbs[2].data, mock_stderr[2]) + def test_real_time_output_streaming_bufsize(self): + # Test various values for bufsize and verify it works / doesn't hang the process + cfg.CONF.set_override(name='stream_output', group='actionrunner', override=True) + + bufsize_values = [-100, -2, -1, 0, 1, 2, 1024, 2048, 4096, 10000] + + for index, bufsize in enumerate(bufsize_values, 1): + cfg.CONF.set_override(name='stream_output_buffer_size', override=bufsize, + group='actionrunner') + + output_dbs = ActionExecutionOutput.get_all() + self.assertEqual(len(output_dbs), (index - 1) * 4) + + runner = self._get_mock_runner_obj() + runner.entry_point = PRINT_TO_STDOUT_STDERR_ACTION + runner.pre_run() + (_, output, _) = runner.run({'stdout_count': 2, 'stderr_count': 2}) + + self.assertEqual(output['stdout'], 'stdout line 0\nstdout line 1\n') + self.assertEqual(output['stderr'], 'stderr line 0\nstderr line 1\n') + self.assertEqual(output['exit_code'], 0) + + output_dbs = ActionExecutionOutput.get_all() + self.assertEqual(len(output_dbs), (index) * 4) + @mock.patch('st2common.util.concurrency.subprocess_popen') def test_stdout_interception_and_parsing(self, mock_popen): values = {'delimiter': ACTION_OUTPUT_RESULT_DELIMITER} diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index deb77d620f..3557ce398e 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -15,9 +15,21 @@ source ./virtualenv/bin/activate python ./st2client/setup.py develop st2 --version +# Clean up old screen log files +rm -f logs/screen-*.log + # start dev environment in screens ./tools/launchdev.sh start -x +# Give processes some time to start and check logs to see if all the services +# started or if there was any error / failure +echo "Giving screen processes some time to start..." +sleep 10 + +echo " === START: Catting screen process log files. ===" +cat logs/screen-*.log +echo " === END: Catting screen process log files. ===" + # This script runs as root on Travis which means other processes which don't run # as root can't write to logs/ directory and tests fail chmod 777 logs/ diff --git a/st2api/st2api/controllers/v1/rule_views.py b/st2api/st2api/controllers/v1/rule_views.py index c607dac0cb..04c1114bad 100644 --- a/st2api/st2api/controllers/v1/rule_views.py +++ b/st2api/st2api/controllers/v1/rule_views.py @@ -99,8 +99,9 @@ def get_all(self, exclude_attributes=None, include_attributes=None, sort=None, o return rules def get_one(self, ref_or_id, requester_user): + from_model_kwargs = {'mask_secrets': True} rule = self._get_one(ref_or_id, permission_type=PermissionType.RULE_VIEW, - requester_user=requester_user) + requester_user=requester_user, from_model_kwargs=from_model_kwargs) result = self._append_view_properties([rule.json])[0] rule.json = result return rule diff --git a/st2api/st2api/controllers/v1/rules.py b/st2api/st2api/controllers/v1/rules.py index d64fb73732..4921b7b279 100644 --- a/st2api/st2api/controllers/v1/rules.py +++ b/st2api/st2api/controllers/v1/rules.py @@ -20,6 +20,7 @@ from st2common import log as logging from st2common.exceptions.apivalidation import ValueValidationException from st2common.exceptions.triggers import TriggerDoesNotExistException +from st2api.controllers.base import BaseRestControllerMixin from st2api.controllers.resource import BaseResourceIsolationControllerMixin from st2api.controllers.resource import ContentPackResourceController from st2api.controllers.controller_transforms import transform_to_bool @@ -39,7 +40,8 @@ LOG = logging.getLogger(__name__) -class RuleController(BaseResourceIsolationControllerMixin, ContentPackResourceController): +class RuleController(BaseRestControllerMixin, BaseResourceIsolationControllerMixin, + ContentPackResourceController): """ Implements the RESTful web endpoint that handles the lifecycle of Rules in the system. @@ -68,8 +70,11 @@ class RuleController(BaseResourceIsolationControllerMixin, ContentPackResourceCo mandatory_include_fields_retrieve = ['pack', 'name', 'trigger'] def get_all(self, exclude_attributes=None, include_attributes=None, sort=None, offset=0, - limit=None, requester_user=None, **raw_filters): - from_model_kwargs = {'ignore_missing_trigger': True} + limit=None, show_secrets=False, requester_user=None, **raw_filters): + from_model_kwargs = { + 'ignore_missing_trigger': True, + 'mask_secrets': self._get_mask_secrets(requester_user, show_secrets=show_secrets) + } return super(RuleController, self)._get_all(exclude_fields=exclude_attributes, include_fields=include_attributes, from_model_kwargs=from_model_kwargs, @@ -79,8 +84,11 @@ def get_all(self, exclude_attributes=None, include_attributes=None, sort=None, o raw_filters=raw_filters, requester_user=requester_user) - def get_one(self, ref_or_id, requester_user): - from_model_kwargs = {'ignore_missing_trigger': True} + def get_one(self, ref_or_id, requester_user, show_secrets=False): + from_model_kwargs = { + 'ignore_missing_trigger': True, + 'mask_secrets': self._get_mask_secrets(requester_user, show_secrets=show_secrets) + } return super(RuleController, self)._get_one(ref_or_id, from_model_kwargs=from_model_kwargs, requester_user=requester_user, permission_type=PermissionType.RULE_VIEW) diff --git a/st2api/tests/unit/controllers/v1/test_action_views.py b/st2api/tests/unit/controllers/v1/test_action_views.py index cd6491b42c..768802c254 100644 --- a/st2api/tests/unit/controllers/v1/test_action_views.py +++ b/st2api/tests/unit/controllers/v1/test_action_views.py @@ -240,7 +240,8 @@ def test_get_one_ref_python_content_type(self): try: get_resp = self.app.get('/v1/actions/views/entry_point/%s' % action_ref) self.assertEqual(get_resp.status_int, 200) - self.assertEqual(get_resp.headers['Content-Type'], 'application/x-python') + self.assertTrue(get_resp.headers['Content-Type'] in ['application/x-python', + 'text/x-python']) finally: self.app.delete('/v1/actions/%s' % action_id) diff --git a/st2api/tests/unit/controllers/v1/test_rules.py b/st2api/tests/unit/controllers/v1/test_rules.py index ae8063095f..3651bef6ac 100644 --- a/st2api/tests/unit/controllers/v1/test_rules.py +++ b/st2api/tests/unit/controllers/v1/test_rules.py @@ -20,6 +20,7 @@ from st2common.constants.rules import RULE_TYPE_STANDARD, RULE_TYPE_BACKSTOP from st2common.constants.pack import DEFAULT_PACK_NAME +from st2common.constants.secrets import MASKED_ATTRIBUTE_VALUE from st2common.persistence.trigger import Trigger from st2common.models.system.common import ResourceReference from st2common.transport.publishers import PoolPublisher @@ -185,6 +186,58 @@ def test_get_all_enabled(self): self.__do_delete(self.__get_rule_id(post_resp_rule_1)) self.__do_delete(self.__get_rule_id(post_resp_rule_3)) + def test_get_all_action_parameters_secrets_masking(self): + post_resp_rule_1 = self.__do_post(RulesControllerTestCase.RULE_1) + + # Verify parameter is masked by default + resp = self.app.get('/v1/rules') + self.assertEqual('action' in resp.json[0], True) + self.assertEqual(resp.json[0]['action']['parameters']['action_secret'], + MASKED_ATTRIBUTE_VALUE) + + # Verify ?show_secrets=true works + resp = self.app.get('/v1/rules?include_attributes=action&show_secrets=true') + self.assertEqual('action' in resp.json[0], True) + self.assertEqual(resp.json[0]['action']['parameters']['action_secret'], 'secret') + + self.__do_delete(self.__get_rule_id(post_resp_rule_1)) + + def test_get_all_parameters_mask_with_exclude_parameters(self): + post_resp_rule_1 = self.__do_post(RulesControllerTestCase.RULE_1) + resp = self.app.get('/v1/rules?exclude_attributes=action') + self.assertEqual('action' in resp.json[0], False) + self.__do_delete(self.__get_rule_id(post_resp_rule_1)) + + def test_get_all_parameters_mask_with_include_parameters(self): + post_resp_rule_1 = self.__do_post(RulesControllerTestCase.RULE_1) + + # Verify parameter is masked by default + resp = self.app.get('/v1/rules?include_attributes=action') + self.assertEqual('action' in resp.json[0], True) + self.assertEqual(resp.json[0]['action']['parameters']['action_secret'], + MASKED_ATTRIBUTE_VALUE) + + # Verify ?show_secrets=true works + resp = self.app.get('/v1/rules?include_attributes=action&show_secrets=true') + self.assertEqual('action' in resp.json[0], True) + self.assertEqual(resp.json[0]['action']['parameters']['action_secret'], 'secret') + + self.__do_delete(self.__get_rule_id(post_resp_rule_1)) + + def test_get_one_action_parameters_secrets_masking(self): + post_resp_rule_1 = self.__do_post(RulesControllerTestCase.RULE_1) + + # Verify parameter is masked by default + resp = self.app.get('/v1/rules/%s' % (post_resp_rule_1.json['id'])) + self.assertEqual(resp.json['action']['parameters']['action_secret'], + MASKED_ATTRIBUTE_VALUE) + + # Verify ?show_secrets=true works + resp = self.app.get('/v1/rules/%s?show_secrets=true' % (post_resp_rule_1.json['id'])) + self.assertEqual(resp.json['action']['parameters']['action_secret'], 'secret') + + self.__do_delete(self.__get_rule_id(post_resp_rule_1)) + def test_get_one_by_id(self): post_resp = self.__do_post(RulesControllerTestCase.RULE_1) rule_id = self.__get_rule_id(post_resp) diff --git a/st2auth/st2auth/controllers/v1/auth.py b/st2auth/st2auth/controllers/v1/auth.py index e32266b9d2..3b3073e32b 100644 --- a/st2auth/st2auth/controllers/v1/auth.py +++ b/st2auth/st2auth/controllers/v1/auth.py @@ -22,6 +22,7 @@ from st2common.router import exc from st2common.router import Response from st2common.util import auth as auth_utils +from st2common.util import api as api_utils from st2common import log as logging import st2auth.handlers as handlers @@ -80,7 +81,8 @@ def post(self, request, **kwargs): def process_successful_response(token): resp = Response(json=token, status=http_client.CREATED) - resp.headers['X-API-URL'] = cfg.CONF.auth.api_url + # NOTE: gunicon fails and throws an error if header value is not a string (e.g. if it's None) + resp.headers['X-API-URL'] = api_utils.get_base_public_api_url() return resp diff --git a/st2auth/tests/unit/controllers/v1/test_token.py b/st2auth/tests/unit/controllers/v1/test_token.py index d1a5e46e5e..3a3eb237b1 100644 --- a/st2auth/tests/unit/controllers/v1/test_token.py +++ b/st2auth/tests/unit/controllers/v1/test_token.py @@ -87,6 +87,7 @@ def _test_token_post(self, path=TOKEN_V1_PATH): actual_expiry = isotime.parse(response.json['expiry']) self.assertLess(timestamp, actual_expiry) self.assertLess(actual_expiry, expected_expiry) + return response def test_token_post_unauthorized(self): response = self.app.post_json(TOKEN_V1_PATH, {}, expect_errors=True, extra_environ={ @@ -109,6 +110,22 @@ def test_token_post_new_user(self): def test_token_post_existing_user(self): self._test_token_post() + @mock.patch.object( + User, 'get_by_name', + mock.MagicMock(return_value=UserDB(name=USERNAME))) + def test_token_post_success_x_api_url_header_value(self): + # auth.api_url option is explicitly set + cfg.CONF.set_override('api_url', override='https://example.com', group='auth') + + resp = self._test_token_post() + self.assertEqual(resp.headers['X-API-URL'], 'https://example.com') + + # auth.api_url option is not set, url is inferred from listen host and port + cfg.CONF.set_override('api_url', override=None, group='auth') + + resp = self._test_token_post() + self.assertEqual(resp.headers['X-API-URL'], 'http://127.0.0.1:9101') + @mock.patch.object( User, 'get_by_name', mock.MagicMock(return_value=UserDB(name=USERNAME))) diff --git a/st2client/st2client/client.py b/st2client/st2client/client.py index 280174edb5..fe1d4cc1b0 100644 --- a/st2client/st2client/client.py +++ b/st2client/st2client/client.py @@ -25,6 +25,7 @@ from st2client.models.core import ResourceManager from st2client.models.core import ActionAliasResourceManager from st2client.models.core import ActionAliasExecutionManager +from st2client.models.core import ActionResourceManager from st2client.models.core import ExecutionResourceManager from st2client.models.core import InquiryResourceManager from st2client.models.core import TriggerInstanceResourceManager @@ -118,7 +119,7 @@ def __init__(self, base_url=None, auth_url=None, api_url=None, stream_url=None, models.Token, self.endpoints['auth'], cacert=self.cacert, debug=self.debug) self.managers['RunnerType'] = ResourceManager( models.RunnerType, self.endpoints['api'], cacert=self.cacert, debug=self.debug) - self.managers['Action'] = ResourceManager( + self.managers['Action'] = ActionResourceManager( models.Action, self.endpoints['api'], cacert=self.cacert, debug=self.debug) self.managers['ActionAlias'] = ActionAliasResourceManager( models.ActionAlias, self.endpoints['api'], cacert=self.cacert, debug=self.debug) diff --git a/st2client/st2client/commands/action.py b/st2client/st2client/commands/action.py index e6b594bde8..f458303bc0 100644 --- a/st2client/st2client/commands/action.py +++ b/st2client/st2client/commands/action.py @@ -296,7 +296,7 @@ def _add_common_options(self): detail_arg_grp = execution_details_arg_grp.add_mutually_exclusive_group() detail_arg_grp.add_argument('--attr', nargs='+', - default=['id', 'status', 'parameters', 'result'], + default=self.display_attributes, help=('List of attributes to include in the ' 'output. "all" or unspecified will ' 'return all attributes.')) @@ -1167,7 +1167,7 @@ def run_and_print(self, args, **kwargs): class ActionExecutionGetCommand(ActionRunCommandMixin, ResourceViewCommand): display_attributes = ['id', 'action.ref', 'context.user', 'parameters', 'status', - 'start_timestamp', 'end_timestamp', 'result', 'liveaction'] + 'start_timestamp', 'end_timestamp', 'result'] include_attributes = ['action.ref', 'action.runner_type', 'start_timestamp', 'end_timestamp'] @@ -1321,7 +1321,7 @@ def run(self, args, **kwargs): class ActionExecutionPauseCommand(ActionRunCommandMixin, ResourceViewCommand): display_attributes = ['id', 'action.ref', 'context.user', 'parameters', 'status', - 'start_timestamp', 'end_timestamp', 'result', 'liveaction'] + 'start_timestamp', 'end_timestamp', 'result'] def __init__(self, resource, *args, **kwargs): super(ActionExecutionPauseCommand, self).__init__( @@ -1364,7 +1364,7 @@ def _print_result(self, args, execution_id, execution, **kwargs): class ActionExecutionResumeCommand(ActionRunCommandMixin, ResourceViewCommand): display_attributes = ['id', 'action.ref', 'context.user', 'parameters', 'status', - 'start_timestamp', 'end_timestamp', 'result', 'liveaction'] + 'start_timestamp', 'end_timestamp', 'result'] def __init__(self, resource, *args, **kwargs): super(ActionExecutionResumeCommand, self).__init__( diff --git a/st2client/st2client/models/action.py b/st2client/st2client/models/action.py index d931c48427..6432192792 100644 --- a/st2client/st2client/models/action.py +++ b/st2client/st2client/models/action.py @@ -33,6 +33,7 @@ class RunnerType(core.Resource): class Action(core.Resource): _plural = 'Actions' _repr_attributes = ['name', 'pack', 'enabled', 'runner_type'] + _url_path = 'actions' class Execution(core.Resource): diff --git a/st2client/st2client/models/core.py b/st2client/st2client/models/core.py index 627b7183aa..0a2738fbbc 100644 --- a/st2client/st2client/models/core.py +++ b/st2client/st2client/models/core.py @@ -376,6 +376,18 @@ def match_and_execute(self, instance, **kwargs): return instance +class ActionResourceManager(ResourceManager): + @add_auth_token_to_kwargs_from_env + def get_entrypoint(self, ref_or_id, **kwargs): + url = '/%s/views/entry_point/%s' % (self.resource.get_url_path_name(), ref_or_id) + + response = self.client.get(url, **kwargs) + if response.status_code != http_client.OK: + self.handle_error(response) + + return response.text + + class ExecutionResourceManager(ResourceManager): @add_auth_token_to_kwargs_from_env def re_run(self, execution_id, parameters=None, tasks=None, no_reset=None, user=None, delay=0, diff --git a/st2client/tests/fixtures/execution_double_backslash.txt b/st2client/tests/fixtures/execution_double_backslash.txt index 2e589baa93..5437c7add4 100644 --- a/st2client/tests/fixtures/execution_double_backslash.txt +++ b/st2client/tests/fixtures/execution_double_backslash.txt @@ -1,7 +1,11 @@ id: 547e19561e2e2417d3dde333 -status: succeeded (1s elapsed) +action.ref: core.local +context.user: stanley parameters: cmd: echo 'C:\Users\ADMINI~1\AppData\Local\Temp\jking_vmware20_test' +status: succeeded (1s elapsed) +start_timestamp: Tue, 02 Dec 2014 19:56:06 UTC +end_timestamp: Tue, 02 Dec 2014 19:56:07 UTC result: failed: false return_code: 0 diff --git a/st2client/tests/fixtures/execution_get_default.txt b/st2client/tests/fixtures/execution_get_default.txt index 2613b61f2c..c29c2c221d 100644 --- a/st2client/tests/fixtures/execution_get_default.txt +++ b/st2client/tests/fixtures/execution_get_default.txt @@ -1,7 +1,11 @@ id: 547e19561e2e2417d3dde398 -status: succeeded (1s elapsed) +action.ref: core.ping +context.user: stanley parameters: cmd: 127.0.0.1 3 +status: succeeded (1s elapsed) +start_timestamp: Tue, 02 Dec 2014 19:56:06 UTC +end_timestamp: Tue, 02 Dec 2014 19:56:07 UTC result: localhost: failed: false diff --git a/st2client/tests/fixtures/execution_get_detail.txt b/st2client/tests/fixtures/execution_get_detail.txt index 9b9edfaabf..4645130ace 100644 --- a/st2client/tests/fixtures/execution_get_detail.txt +++ b/st2client/tests/fixtures/execution_get_detail.txt @@ -28,8 +28,4 @@ | | rtt min/avg/max/mdev = 0.015/0.023/0.030/0.006 ms" | | | } | | | } | -| liveaction | { | -| | "callback": {}, | -| | "id": "1" | -| | } | +-----------------+--------------------------------------------------------------+ diff --git a/st2client/tests/fixtures/execution_get_has_schema.txt b/st2client/tests/fixtures/execution_get_has_schema.txt index 2613b61f2c..c29c2c221d 100644 --- a/st2client/tests/fixtures/execution_get_has_schema.txt +++ b/st2client/tests/fixtures/execution_get_has_schema.txt @@ -1,7 +1,11 @@ id: 547e19561e2e2417d3dde398 -status: succeeded (1s elapsed) +action.ref: core.ping +context.user: stanley parameters: cmd: 127.0.0.1 3 +status: succeeded (1s elapsed) +start_timestamp: Tue, 02 Dec 2014 19:56:06 UTC +end_timestamp: Tue, 02 Dec 2014 19:56:07 UTC result: localhost: failed: false diff --git a/st2client/tests/fixtures/execution_result_has_carriage_return.txt b/st2client/tests/fixtures/execution_result_has_carriage_return.txt index 2484678896..efdbac7e8f 100644 --- a/st2client/tests/fixtures/execution_result_has_carriage_return.txt +++ b/st2client/tests/fixtures/execution_result_has_carriage_return.txt @@ -64,8 +64,4 @@ | | "stdout": "" | | | } | | | } | -| liveaction | { | -| | "callback": {}, | -| | "id": "1" | -| | } | +-----------------+--------------------------------------------------------------+ diff --git a/st2client/tests/fixtures/execution_result_has_carriage_return_py3.txt b/st2client/tests/fixtures/execution_result_has_carriage_return_py3.txt index 1dfa1c59ba..7a10e63a52 100644 --- a/st2client/tests/fixtures/execution_result_has_carriage_return_py3.txt +++ b/st2client/tests/fixtures/execution_result_has_carriage_return_py3.txt @@ -64,8 +64,4 @@ | | "stdout": "" | | | } | | | } | -| liveaction | { | -| | "callback": {}, | -| | "id": "1" | -| | } | +-----------------+--------------------------------------------------------------+ diff --git a/st2client/tests/fixtures/execution_unescape_newline.txt b/st2client/tests/fixtures/execution_unescape_newline.txt index 40ab18deca..4abac251a5 100644 --- a/st2client/tests/fixtures/execution_unescape_newline.txt +++ b/st2client/tests/fixtures/execution_unescape_newline.txt @@ -1,6 +1,10 @@ id: 547e19561e2e2417d3dde123 -status: succeeded (1s elapsed) +action.ref: core.stacktrace +context.user: stanley parameters: None +status: succeeded (1s elapsed) +start_timestamp: Tue, 02 Dec 2014 19:56:06 UTC +end_timestamp: Tue, 02 Dec 2014 19:56:07 UTC result: localhost: failed: false diff --git a/st2client/tests/fixtures/execution_unicode.txt b/st2client/tests/fixtures/execution_unicode.txt index f7209afaa9..7b7491d3b7 100644 --- a/st2client/tests/fixtures/execution_unicode.txt +++ b/st2client/tests/fixtures/execution_unicode.txt @@ -1,7 +1,11 @@ id: 547e19561e2e2417d3dde321 -status: succeeded (1s elapsed) +action.ref: core.local +context.user: stanley parameters: cmd: "echo '‡'" +status: succeeded (1s elapsed) +start_timestamp: Tue, 02 Dec 2014 19:56:06 UTC +end_timestamp: Tue, 02 Dec 2014 19:56:07 UTC result: failed: false return_code: 0 diff --git a/st2client/tests/fixtures/execution_unicode_py3.txt b/st2client/tests/fixtures/execution_unicode_py3.txt index ba96a99346..0db50aa746 100644 --- a/st2client/tests/fixtures/execution_unicode_py3.txt +++ b/st2client/tests/fixtures/execution_unicode_py3.txt @@ -1,7 +1,11 @@ id: 547e19561e2e2417d3dde321 -status: succeeded (1s elapsed) +action.ref: core.local +context.user: stanley parameters: cmd: "echo '\u2021'" +status: succeeded (1s elapsed) +start_timestamp: Tue, 02 Dec 2014 19:56:06 UTC +end_timestamp: Tue, 02 Dec 2014 19:56:07 UTC result: failed: false return_code: 0 diff --git a/st2client/tests/unit/test_client_actions.py b/st2client/tests/unit/test_client_actions.py new file mode 100644 index 0000000000..5a5c798360 --- /dev/null +++ b/st2client/tests/unit/test_client_actions.py @@ -0,0 +1,104 @@ +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import json +import logging +import mock +import unittest2 + +from tests import base + +from st2client import client +from st2client.utils import httpclient + + +LOG = logging.getLogger(__name__) + + +EXECUTION = { + "id": 12345, + "action": { + "ref": "mock.foobar" + }, + "status": "failed", + "result": "non-empty" +} + +ENTRYPOINT = ( + "version: 1.0" + + "description: A basic workflow that runs an arbitrary linux command." + + "input:" + " - cmd" + " - timeout" + + "tasks:" + " task1:" + " action: core.local cmd=<% ctx(cmd) %> timeout=<% ctx(timeout) %>" + " next:" + " - when: <% succeeded() %>" + " publish:" + " - stdout: <% result().stdout %>" + " - stderr: <% result().stderr %>" + + "output:" + " - stdout: <% ctx(stdout) %>" +) + + +class TestActionResourceManager(unittest2.TestCase): + + @classmethod + def setUpClass(cls): + super(TestActionResourceManager, cls).setUpClass() + cls.client = client.Client() + + @mock.patch.object( + httpclient.HTTPClient, 'get', + mock.MagicMock(return_value=base.FakeResponse(json.dumps(ENTRYPOINT), 200, 'OK'))) + def test_get_action_entry_point_by_ref(self): + actual_entrypoint = self.client.actions.get_entrypoint(EXECUTION['action']['ref']) + actual_entrypoint = json.loads(actual_entrypoint) + + endpoint = '/actions/views/entry_point/%s' % EXECUTION['action']['ref'] + httpclient.HTTPClient.get.assert_called_with(endpoint) + + self.assertEqual(ENTRYPOINT, actual_entrypoint) + + @mock.patch.object( + httpclient.HTTPClient, 'get', + mock.MagicMock(return_value=base.FakeResponse(json.dumps(ENTRYPOINT), 200, 'OK'))) + def test_get_action_entry_point_by_id(self): + actual_entrypoint = self.client.actions.get_entrypoint(EXECUTION['id']) + actual_entrypoint = json.loads(actual_entrypoint) + + endpoint = '/actions/views/entry_point/%s' % EXECUTION['id'] + httpclient.HTTPClient.get.assert_called_with(endpoint) + + self.assertEqual(ENTRYPOINT, actual_entrypoint) + + @mock.patch.object( + httpclient.HTTPClient, 'get', + mock.MagicMock(return_value=base.FakeResponse( + json.dumps({}), 404, '404 Client Error: Not Found' + ))) + def test_get_non_existent_action_entry_point(self): + with self.assertRaisesRegexp(Exception, '404 Client Error: Not Found'): + self.client.actions.get_entrypoint('nonexistentpack.nonexistentaction') + + endpoint = '/actions/views/entry_point/%s' % 'nonexistentpack.nonexistentaction' + httpclient.HTTPClient.get.assert_called_with(endpoint) diff --git a/st2client/tests/unit/test_formatters.py b/st2client/tests/unit/test_formatters.py index e9ef0971b8..40289d6423 100644 --- a/st2client/tests/unit/test_formatters.py +++ b/st2client/tests/unit/test_formatters.py @@ -116,7 +116,9 @@ def test_execution_get_default_in_json(self): argv = ['execution', 'get', EXECUTION['id'], '-j'] content = self._get_execution(argv) self.assertEqual(json.loads(content), - jsutil.get_kvps(EXECUTION, ['id', 'status', 'parameters', 'result'])) + jsutil.get_kvps(EXECUTION, ['id', 'action.ref', 'context.user', + 'start_timestamp', 'end_timestamp', 'status', + 'parameters', 'result'])) def test_execution_get_detail(self): argv = ['execution', 'get', EXECUTION['id'], '-d'] @@ -181,6 +183,9 @@ def test_execution_get_detail_in_json(self): # Sufficient to check if output contains all expected keys. The entire result will not # match as content will contain characters which improve rendering. for k in six.iterkeys(EXECUTION): + if k in ['liveaction', 'callback']: + continue + if k in content: continue self.assertTrue(False, 'Missing key %s. %s != %s' % (k, EXECUTION, content_dict)) diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index 1b31b93e77..4b89c08650 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -374,9 +374,19 @@ def register_opts(ignore_errors=False): 'virtualenv_opts', default=['--system-site-packages'], help='List of virtualenv options to be passsed to "virtualenv" command that ' 'creates pack virtualenv.'), + cfg.ListOpt( + 'pip_opts', default=[], + help='List of pip options to be passed to "pip install" command when installing pack ' + 'dependencies into pack virtual environment.'), cfg.BoolOpt( 'stream_output', default=True, - help='True to store and stream action output (stdout and stderr) in real-time.') + help='True to store and stream action output (stdout and stderr) in real-time.'), + cfg.IntOpt( + 'stream_output_buffer_size', default=-1, + help=('Buffer size to use for real time action output streaming. 0 means unbuffered ' + '1 means line buffered, -1 means system default, which usually means fully ' + 'buffered and any other positive value means use a buffer of (approximately) ' + 'that size')) ] do_register_opts(action_runner_opts, group='actionrunner') diff --git a/st2common/st2common/log.py b/st2common/st2common/log.py index cf1347ea81..15c25002a5 100644 --- a/st2common/st2common/log.py +++ b/st2common/st2common/log.py @@ -181,10 +181,22 @@ def _redirect_stderr(): sys.stderr = LoggingStream('STDERR') -def setup(config_file, redirect_stderr=True, excludes=None, disable_existing_loggers=False): +def setup(config_file, redirect_stderr=True, excludes=None, disable_existing_loggers=False, + st2_conf_path=None): """ Configure logging from file. + + :param st2_conf_path: Optional path to st2.conf file. If provided and "config_file" path is + relative to st2.conf path, the config_file path will get resolved to full + absolute path relative to st2.conf. + :type st2_conf_path: ``str`` """ + if st2_conf_path and config_file[:2] == './' and not os.path.isfile(config_file): + # Logging config path is relative to st2.conf, resolve it to full absolute path + directory = os.path.dirname(st2_conf_path) + config_file_name = os.path.basename(config_file) + config_file = os.path.join(directory, config_file_name) + try: logging.config.fileConfig(config_file, defaults=None, diff --git a/st2common/st2common/models/db/rule.py b/st2common/st2common/models/db/rule.py index 7492b5abaa..e56d66cea9 100644 --- a/st2common/st2common/models/db/rule.py +++ b/st2common/st2common/models/db/rule.py @@ -13,11 +13,15 @@ # limitations under the License. from __future__ import absolute_import +import copy import mongoengine as me +from st2common.persistence.action import Action from st2common.models.db import MongoDBAccess from st2common.models.db import stormbase from st2common.constants.types import ResourceType +from st2common.util.secrets import get_secret_parameters +from st2common.util.secrets import mask_secret_parameters class RuleTypeDB(stormbase.StormBaseDB): @@ -101,6 +105,55 @@ class RuleDB(stormbase.StormFoundationDB, stormbase.TagsMixin, stormbase.UIDFieldMixin.get_indexes()) } + def mask_secrets(self, value): + """ + Process the model dictionary and mask secret values. + + NOTE: This method results in one addition "get one" query where we retrieve corresponding + action model so we can correctly mask secret parameters. + + :type value: ``dict`` + :param value: Document dictionary. + + :rtype: ``dict`` + """ + result = copy.deepcopy(value) + + action_ref = result.get('action', {}).get('ref', None) + + if not action_ref: + return result + + action_db = self._get_referenced_action_model(action_ref=action_ref) + + if not action_db: + return result + + secret_parameters = get_secret_parameters(parameters=action_db.parameters) + result['action']['parameters'] = mask_secret_parameters( + parameters=result['action']['parameters'], + secret_parameters=secret_parameters) + + return result + + def _get_referenced_action_model(self, action_ref): + """ + Return Action object for the action referenced in a rule. + + :param action_ref: Action reference. + :type action_ref: ``str`` + + :rtype: ``ActionDB`` + """ + # NOTE: We need to retrieve pack and name since that's needed for the PK + action_dbs = Action.query(only_fields=['pack', 'ref', 'name', 'parameters'], + ref=action_ref, limit=1) + + if action_dbs: + return action_dbs[0] + + return None + def __init__(self, *args, **values): super(RuleDB, self).__init__(*args, **values) self.ref = self.get_reference().ref diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index 91e299ff05..0896532f74 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -2649,6 +2649,10 @@ paths: in: query description: Enabled filter type: string + - name: show_secrets + in: query + description: Show secrets in plain text + type: boolean x-parameters: - name: user in: context @@ -2710,6 +2714,10 @@ paths: description: Entity reference or id type: string required: true + - name: show_secrets + in: query + description: Show secrets in plain text + type: boolean x-parameters: - name: user in: context diff --git a/st2common/st2common/openapi.yaml.j2 b/st2common/st2common/openapi.yaml.j2 index 253f45cfde..3387564e8a 100644 --- a/st2common/st2common/openapi.yaml.j2 +++ b/st2common/st2common/openapi.yaml.j2 @@ -2645,6 +2645,10 @@ paths: in: query description: Enabled filter type: string + - name: show_secrets + in: query + description: Show secrets in plain text + type: boolean x-parameters: - name: user in: context @@ -2706,6 +2710,10 @@ paths: description: Entity reference or id type: string required: true + - name: show_secrets + in: query + description: Show secrets in plain text + type: boolean x-parameters: - name: user in: context diff --git a/st2common/st2common/util/concurrency.py b/st2common/st2common/util/concurrency.py index 62544eec3a..c40d2c311d 100644 --- a/st2common/st2common/util/concurrency.py +++ b/st2common/st2common/util/concurrency.py @@ -41,7 +41,9 @@ 'spawn', 'wait', 'cancel', - 'kill' + 'kill', + 'sleep', + 'get_greenlet_exit_exception_class' ] @@ -111,3 +113,21 @@ def kill(green_thread, *args, **kwargs): return green_thread.kill(*args, **kwargs) else: raise ValueError('Unsupported concurrency library') + + +def sleep(*args, **kwargs): + if CONCURRENCY_LIBRARY == 'eventlet': + return eventlet.sleep(*args, **kwargs) + elif CONCURRENCY_LIBRARY == 'gevent': + return gevent.sleep(*args, **kwargs) + else: + raise ValueError('Unsupported concurrency library') + + +def get_greenlet_exit_exception_class(): + if CONCURRENCY_LIBRARY == 'eventlet': + return eventlet.support.greenlets.GreenletExit + elif CONCURRENCY_LIBRARY == 'gevent': + return gevent.GreenletExit + else: + raise ValueError('Unsupported concurrency library') diff --git a/st2common/st2common/util/green/shell.py b/st2common/st2common/util/green/shell.py index fe286f07eb..d126675251 100644 --- a/st2common/st2common/util/green/shell.py +++ b/st2common/st2common/util/green/shell.py @@ -38,7 +38,8 @@ def run_command(cmd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, cwd=None, env=None, timeout=60, preexec_func=None, kill_func=None, read_stdout_func=None, read_stderr_func=None, - read_stdout_buffer=None, read_stderr_buffer=None, stdin_value=None): + read_stdout_buffer=None, read_stderr_buffer=None, stdin_value=None, + bufsize=0): """ Run the provided command in a subprocess and wait until it completes. @@ -82,6 +83,8 @@ def run_command(cmd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, using live read mode. :type read_stdout_func: ``func`` + :param bufsize: Buffer size argument to pass to subprocess.popen function. + :type bufsize: ``int`` :rtype: ``tuple`` (exit_code, stdout, stderr, timed_out) """ @@ -107,7 +110,8 @@ def run_command(cmd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, # GreenPipe so it doesn't block LOG.debug('Creating subprocess.') process = concurrency.subprocess_popen(args=cmd, stdin=stdin, stdout=stdout, stderr=stderr, - env=env, cwd=cwd, shell=shell, preexec_fn=preexec_func) + env=env, cwd=cwd, shell=shell, preexec_fn=preexec_func, + bufsize=bufsize) if read_stdout_func: LOG.debug('Spawning read_stdout_func function') diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py index b1eacad31d..0ac1c5977c 100644 --- a/st2common/st2common/util/virtualenvs.py +++ b/st2common/st2common/util/virtualenvs.py @@ -158,7 +158,7 @@ def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_se python_binary = cfg.CONF.actionrunner.python_binary python3_binary = cfg.CONF.actionrunner.python3_binary virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary - virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts + virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts or [] if not os.path.isfile(python_binary): raise Exception('Python binary "%s" doesn\'t exist' % (python_binary)) @@ -237,6 +237,7 @@ def install_requirements(virtualenv_path, requirements_file_path, proxy_config=N """ logger = logger or LOG pip_path = os.path.join(virtualenv_path, 'bin/pip') + pip_opts = cfg.CONF.actionrunner.pip_opts or [] cmd = [pip_path] if proxy_config: @@ -253,7 +254,10 @@ def install_requirements(virtualenv_path, requirements_file_path, proxy_config=N if cert: cmd.extend(['--cert', cert]) - cmd.extend(['install', '-U', '-r', requirements_file_path]) + cmd.append('install') + cmd.extend(pip_opts) + cmd.extend(['-U', '-r', requirements_file_path]) + env = get_env_for_subprocess_command() logger.debug('Installing requirements from file %s with command %s.', @@ -278,6 +282,7 @@ def install_requirement(virtualenv_path, requirement, proxy_config=None, logger= """ logger = logger or LOG pip_path = os.path.join(virtualenv_path, 'bin/pip') + pip_opts = cfg.CONF.actionrunner.pip_opts or [] cmd = [pip_path] if proxy_config: @@ -294,7 +299,9 @@ def install_requirement(virtualenv_path, requirement, proxy_config=None, logger= if cert: cmd.extend(['--cert', cert]) - cmd.extend(['install', requirement]) + cmd.append('install') + cmd.extend(pip_opts) + cmd.extend([requirement]) env = get_env_for_subprocess_command() logger.debug('Installing requirement %s with command %s.', requirement, ' '.join(cmd)) diff --git a/st2common/tests/unit/test_db.py b/st2common/tests/unit/test_db.py index 8db54e7cf4..462ab8fa3e 100644 --- a/st2common/tests/unit/test_db.py +++ b/st2common/tests/unit/test_db.py @@ -21,7 +21,6 @@ import mongoengine.connection from oslo_config import cfg from pymongo.errors import ConnectionFailure - from st2common.constants.triggers import TRIGGER_INSTANCE_PROCESSED from st2common.models.system.common import ResourceReference from st2common.transport.publishers import PoolPublisher @@ -521,6 +520,10 @@ def _delete(model_objects): "p3": { "type": "boolean", "default": False + }, + "p4": { + "type": "string", + "secret": True } }, "additionalProperties": False @@ -661,12 +664,13 @@ def _create_save_action(runnertype, metadata=False): runner_type={'name': runnertype.name}) if not metadata: - created.parameters = {'p1': None, 'p2': None, 'p3': None} + created.parameters = {'p1': None, 'p2': None, 'p3': None, 'p4': None} else: created.parameters = { 'p1': {'type': 'string', 'required': True}, 'p2': {'type': 'number', 'default': 2868}, - 'p3': {'type': 'boolean', 'default': False} + 'p3': {'type': 'boolean', 'default': False}, + 'p4': {'type': 'string', 'secret': True} } return Action.add_or_update(created) diff --git a/st2reactor/st2reactor/cmd/timersengine.py b/st2reactor/st2reactor/cmd/timersengine.py index 71d3fb29c5..b91dcaefcb 100644 --- a/st2reactor/st2reactor/cmd/timersengine.py +++ b/st2reactor/st2reactor/cmd/timersengine.py @@ -13,13 +13,14 @@ # limitations under the License. from __future__ import absolute_import + import os import sys -import eventlet from oslo_config import cfg from st2common import log as logging +from st2common.util import concurrency from st2common.constants.timer import TIMER_ENABLED_LOG_LINE, TIMER_DISABLED_LOG_LINE from st2common.logging.misc import get_logger_name_for_module from st2common.service_setup import setup as common_setup @@ -61,7 +62,7 @@ def _run_worker(): if cfg.CONF.timer.enable or cfg.CONF.timersengine.enable: local_tz = cfg.CONF.timer.local_timezone or cfg.CONF.timersengine.local_timezone timer = St2Timer(local_timezone=local_tz) - timer_thread = eventlet.spawn(_kickoff_timer, timer) + timer_thread = concurrency.spawn(_kickoff_timer, timer) LOG.info(TIMER_ENABLED_LOG_LINE) return timer_thread.wait() else: @@ -84,7 +85,7 @@ def main(): return _run_worker() except SystemExit as exit_code: sys.exit(exit_code) - except: + except Exception: LOG.exception('(PID=%s) TimerEngine quit due to exception.', os.getpid()) return 1 finally: diff --git a/st2reactor/st2reactor/container/manager.py b/st2reactor/st2reactor/container/manager.py index 917481ddd0..6942c78a81 100644 --- a/st2reactor/st2reactor/container/manager.py +++ b/st2reactor/st2reactor/container/manager.py @@ -13,13 +13,13 @@ # limitations under the License. from __future__ import absolute_import + import os import sys import signal -import eventlet - from st2common import log as logging +from st2common.util import concurrency from st2reactor.container.process_container import ProcessSensorContainer from st2common.services.sensor_watcher import SensorWatcher from st2common.models.system.common import ResourceReference @@ -75,7 +75,7 @@ def _spin_container_and_wait(self, sensors): self._sensor_container = ProcessSensorContainer( sensors=sensors, single_sensor_mode=self._single_sensor_mode) - self._container_thread = eventlet.spawn(self._sensor_container.run) + self._container_thread = concurrency.spawn(self._sensor_container.run) LOG.debug('Starting sensor CUD watcher...') self._sensors_watcher.start() @@ -90,7 +90,7 @@ def _spin_container_and_wait(self, sensors): LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(), sys.exc_info()[0].__name__) - eventlet.kill(self._container_thread) + concurrency.kill(self._container_thread) self._container_thread = None return exit_code diff --git a/st2reactor/st2reactor/container/process_container.py b/st2reactor/st2reactor/container/process_container.py index 81fe9654ac..592447b811 100644 --- a/st2reactor/st2reactor/container/process_container.py +++ b/st2reactor/st2reactor/container/process_container.py @@ -13,6 +13,7 @@ # limitations under the License. from __future__ import absolute_import + import os import sys import time @@ -22,11 +23,10 @@ from collections import defaultdict import six -import eventlet -from eventlet.support import greenlets as greenlet from oslo_config import cfg from st2common import log as logging +from st2common.util import concurrency from st2common.constants.error_messages import PACK_VIRTUALENV_DOESNT_EXIST from st2common.constants.error_messages import PACK_VIRTUALENV_USES_PYTHON3 from st2common.constants.system import API_URL_ENV_VARIABLE_NAME @@ -79,16 +79,26 @@ class ProcessSensorContainer(object): Sensor container which runs sensors in a separate process. """ - def __init__(self, sensors, poll_interval=5, single_sensor_mode=False, dispatcher=None): + def __init__(self, sensors, poll_interval=5, single_sensor_mode=False, dispatcher=None, + wrapper_script_path=WRAPPER_SCRIPT_PATH, create_token=True): """ :param sensors: A list of sensor dicts. :type sensors: ``list`` of ``dict`` :param poll_interval: How long to sleep between each poll for running / dead sensors. :type poll_interval: ``float`` + + :param wrapper_script_path: Path to the sensor wrapper script. + :type wrapper_script_path: ``str`` + + :param create_token: True to create temporary authentication token for the purpose for each + sensor process and add it to that process environment variables. + :type create_token: ``bool`` """ self._poll_interval = poll_interval self._single_sensor_mode = single_sensor_mode + self._wrapper_script_path = wrapper_script_path + self._create_token = create_token if self._single_sensor_mode: # For more immediate feedback we use lower poll interval when running in single sensor @@ -98,9 +108,7 @@ def __init__(self, sensors, poll_interval=5, single_sensor_mode=False, dispatche self._sensors = {} # maps sensor_id -> sensor object self._processes = {} # maps sensor_id -> sensor process - if not dispatcher: - dispatcher = TriggerDispatcher(LOG) - self._dispatcher = dispatcher + self._dispatcher = dispatcher or TriggerDispatcher(LOG) self._stopped = False self._exit_code = None # exit code with which this process should exit @@ -129,6 +137,8 @@ def __init__(self, sensors, poll_interval=5, single_sensor_mode=False, dispatche def run(self): self._run_all_sensors() + success_exception_cls = concurrency.get_greenlet_exit_exception_class() + try: while not self._stopped: # Poll for all running processes @@ -140,8 +150,8 @@ def run(self): else: LOG.debug('No active sensors') - eventlet.sleep(self._poll_interval) - except greenlet.GreenletExit: + concurrency.sleep(self._poll_interval) + except success_exception_cls: # This exception is thrown when sensor container manager # kills the thread which runs process container. Not sure # if this is the best thing to do. @@ -180,8 +190,8 @@ def _poll_sensors_for_results(self, sensor_ids): # Try to respawn a dead process (maybe it was a simple failure which can be # resolved with a restart) - eventlet.spawn_n(self._respawn_sensor, sensor_id=sensor_id, sensor=sensor, - exit_code=status) + concurrency.spawn(self._respawn_sensor, sensor_id=sensor_id, sensor=sensor, + exit_code=status) else: sensor_start_time = self._sensor_start_times[sensor_id] sensor_respawn_count = self._sensor_respawn_counts[sensor_id] @@ -294,26 +304,7 @@ def _spawn_sensor_process(self, sensor): msg = PACK_VIRTUALENV_USES_PYTHON3 % format_values raise Exception(msg) - trigger_type_refs = sensor['trigger_types'] or [] - trigger_type_refs = ','.join(trigger_type_refs) - - parent_args = json.dumps(sys.argv[1:]) - - args = [ - python_path, - WRAPPER_SCRIPT_PATH, - '--pack=%s' % (sensor['pack']), - '--file-path=%s' % (sensor['file_path']), - '--class-name=%s' % (sensor['class_name']), - '--trigger-type-refs=%s' % (trigger_type_refs), - '--parent-args=%s' % (parent_args) - ] - - if sensor['poll_interval']: - args.append('--poll-interval=%s' % (sensor['poll_interval'])) - - sandbox_python_path = get_sandbox_python_path(inherit_from_parent=True, - inherit_parent_virtualenv=True) + args = self._get_args_for_wrapper_script(python_binary=python_path, sensor=sensor) if self._enable_common_pack_libs: pack_common_libs_path = get_pack_common_libs_path_for_pack_ref(pack_ref=pack_ref) @@ -322,27 +313,34 @@ def _spawn_sensor_process(self, sensor): env = os.environ.copy() + sandbox_python_path = get_sandbox_python_path(inherit_from_parent=True, + inherit_parent_virtualenv=True) + if self._enable_common_pack_libs and pack_common_libs_path: env['PYTHONPATH'] = pack_common_libs_path + ':' + sandbox_python_path else: env['PYTHONPATH'] = sandbox_python_path - # Include full api URL and API token specific to that sensor - ttl = cfg.CONF.auth.service_token_ttl - metadata = { - 'service': 'sensors_container', - 'sensor_path': sensor['file_path'], - 'sensor_class': sensor['class_name'] - } - temporary_token = create_token(username='sensors_container', ttl=ttl, metadata=metadata, - service=True) + if self._create_token: + # Include full api URL and API token specific to that sensor + LOG.debug('Creating temporary auth token for sensor %s' % (sensor['class_name'])) - env[API_URL_ENV_VARIABLE_NAME] = get_full_public_api_url() - env[AUTH_TOKEN_ENV_VARIABLE_NAME] = temporary_token.token + ttl = cfg.CONF.auth.service_token_ttl + metadata = { + 'service': 'sensors_container', + 'sensor_path': sensor['file_path'], + 'sensor_class': sensor['class_name'] + } + temporary_token = create_token(username='sensors_container', ttl=ttl, metadata=metadata, + service=True) + + env[API_URL_ENV_VARIABLE_NAME] = get_full_public_api_url() + env[AUTH_TOKEN_ENV_VARIABLE_NAME] = temporary_token.token + + # TODO 1: Purge temporary token when service stops or sensor process dies + # TODO 2: Store metadata (wrapper process id) with the token and delete + # tokens for old, dead processes on startup - # TODO 1: Purge temporary token when service stops or sensor process dies - # TODO 2: Store metadata (wrapper process id) with the token and delete - # tokens for old, dead processes on startup cmd = ' '.join(args) LOG.debug('Running sensor subprocess (cmd="%s")', cmd) @@ -434,7 +432,7 @@ def _respawn_sensor(self, sensor_id, sensor, exit_code): self._sensor_respawn_counts[sensor_id] += 1 sleep_delay = (SENSOR_RESPAWN_DELAY * self._sensor_respawn_counts[sensor_id]) - eventlet.sleep(sleep_delay) + concurrency.sleep(sleep_delay) try: self._spawn_sensor_process(sensor=sensor) @@ -459,6 +457,38 @@ def _should_respawn_sensor(self, sensor_id, sensor, exit_code): return True + def _get_args_for_wrapper_script(self, python_binary, sensor): + """ + Return CLI arguments passed to the sensor wrapper script. + + :param python_binary: Python binary used to execute wrapper script. + :type python_binary: ``str`` + + :param sensor: Sensor object dictionary. + :type sensor: ``dict`` + + :rtype: ``list`` + """ + trigger_type_refs = sensor['trigger_types'] or [] + trigger_type_refs = ','.join(trigger_type_refs) + + parent_args = json.dumps(sys.argv[1:]) + + args = [ + python_binary, + self._wrapper_script_path, + '--pack=%s' % (sensor['pack']), + '--file-path=%s' % (sensor['file_path']), + '--class-name=%s' % (sensor['class_name']), + '--trigger-type-refs=%s' % (trigger_type_refs), + '--parent-args=%s' % (parent_args) + ] + + if sensor['poll_interval']: + args.append('--poll-interval=%s' % (sensor['poll_interval'])) + + return args + def _get_sensor_id(self, sensor): """ Return unique identifier for the provider sensor dict. diff --git a/st2reactor/st2reactor/garbage_collector/base.py b/st2reactor/st2reactor/garbage_collector/base.py index ae9c747b6f..c1d4c14ab1 100644 --- a/st2reactor/st2reactor/garbage_collector/base.py +++ b/st2reactor/st2reactor/garbage_collector/base.py @@ -23,11 +23,10 @@ import random import six -import eventlet -from eventlet.support import greenlets as greenlet from oslo_config import cfg from st2common import log as logging +from st2common.util import concurrency from st2common.constants.exit_codes import SUCCESS_EXIT_CODE from st2common.constants.exit_codes import FAILURE_EXIT_CODE from st2common.constants.garbage_collection import DEFAULT_COLLECTION_INTERVAL @@ -81,11 +80,13 @@ def run(self): # Wait a couple of seconds before performing initial collection to prevent thundering herd # effect when restarting multiple services at the same time jitter_seconds = random.uniform(0, 3) - eventlet.sleep(jitter_seconds) + concurrency.sleep(jitter_seconds) + + success_exception_cls = concurrency.get_greenlet_exit_exception_class() try: self._main_loop() - except greenlet.GreenletExit: + except success_exception_cls: self._running = False return SUCCESS_EXIT_CODE except Exception as e: @@ -111,7 +112,7 @@ def _main_loop(self): LOG.info('Sleeping for %s seconds before next garbage collection...' % (self._collection_interval)) - eventlet.sleep(self._collection_interval) + concurrency.sleep(self._collection_interval) def _validate_ttl_values(self): """ @@ -142,7 +143,7 @@ def _perform_garbage_collection(self): if self._action_executions_ttl and self._action_executions_ttl >= MINIMUM_TTL_DAYS: LOG.info(proc_message, obj_type) self._purge_action_executions() - eventlet.sleep(self._sleep_delay) + concurrency.sleep(self._sleep_delay) else: LOG.debug(skip_message, obj_type) @@ -151,7 +152,7 @@ def _perform_garbage_collection(self): self._action_executions_output_ttl >= MINIMUM_TTL_DAYS_EXECUTION_OUTPUT: LOG.info(proc_message, obj_type) self._purge_action_executions_output() - eventlet.sleep(self._sleep_delay) + concurrency.sleep(self._sleep_delay) else: LOG.debug(skip_message, obj_type) @@ -159,7 +160,7 @@ def _perform_garbage_collection(self): if self._trigger_instances_ttl and self._trigger_instances_ttl >= MINIMUM_TTL_DAYS: LOG.info(proc_message, obj_type) self._purge_trigger_instances() - eventlet.sleep(self._sleep_delay) + concurrency.sleep(self._sleep_delay) else: LOG.debug(skip_message, obj_type) @@ -167,7 +168,7 @@ def _perform_garbage_collection(self): if self._purge_inquiries: LOG.info(proc_message, obj_type) self._timeout_inquiries() - eventlet.sleep(self._sleep_delay) + concurrency.sleep(self._sleep_delay) else: LOG.debug(skip_message, obj_type) @@ -175,7 +176,7 @@ def _perform_garbage_collection(self): if self._workflow_execution_max_idle > 0: LOG.info(proc_message, obj_type) self._purge_orphaned_workflow_executions() - eventlet.sleep(self._sleep_delay) + concurrency.sleep(self._sleep_delay) else: LOG.debug(skip_message, obj_type) diff --git a/st2reactor/st2reactor/sensor/base.py b/st2reactor/st2reactor/sensor/base.py index 617ed565a6..f7350c06e3 100644 --- a/st2reactor/st2reactor/sensor/base.py +++ b/st2reactor/st2reactor/sensor/base.py @@ -13,10 +13,12 @@ # limitations under the License. from __future__ import absolute_import + import abc import six -import eventlet + +from st2common.util import concurrency __all__ = [ 'Sensor', @@ -117,7 +119,7 @@ def poll(self): def run(self): while True: self.poll() - eventlet.sleep(self._poll_interval) + concurrency.sleep(self._poll_interval) def get_poll_interval(self): """ diff --git a/st2reactor/tests/integration/test_garbage_collector.py b/st2reactor/tests/integration/test_garbage_collector.py index d91d8016f2..f321025911 100644 --- a/st2reactor/tests/integration/test_garbage_collector.py +++ b/st2reactor/tests/integration/test_garbage_collector.py @@ -19,9 +19,7 @@ import signal import datetime -import eventlet -from eventlet.green import subprocess - +from st2common.util import concurrency from st2common.constants import action as action_constants from st2common.util import date as date_utils from st2common.models.db.execution import ActionExecutionDB @@ -187,7 +185,7 @@ def test_garbage_collection(self): process = self._start_garbage_collector() # Give it some time to perform garbage collection and kill it - eventlet.sleep(15) + concurrency.sleep(15) process.send_signal(signal.SIGKILL) self.remove_process(process=process) @@ -235,7 +233,7 @@ def test_inquiry_garbage_collection(self): process = self._start_garbage_collector() # Give it some time to perform garbage collection and kill it - eventlet.sleep(15) + concurrency.sleep(15) process.send_signal(signal.SIGKILL) self.remove_process(process=process) @@ -254,6 +252,7 @@ def _create_inquiry(self, ttl, timestamp): executions.create_execution_object(liveaction_db) def _start_garbage_collector(self): + subprocess = concurrency.get_subprocess_module() process = subprocess.Popen(CMD_INQUIRY, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, preexec_fn=os.setsid) self.add_process(process=process) diff --git a/st2reactor/tests/integration/test_rules_engine.py b/st2reactor/tests/integration/test_rules_engine.py index f61709e66c..ab7526925c 100644 --- a/st2reactor/tests/integration/test_rules_engine.py +++ b/st2reactor/tests/integration/test_rules_engine.py @@ -19,8 +19,7 @@ import signal import tempfile -from eventlet.green import subprocess - +from st2common.util import concurrency from st2common.constants.timer import TIMER_ENABLED_LOG_LINE from st2common.constants.timer import TIMER_DISABLED_LOG_LINE from st2tests.base import IntegrationTestCase @@ -134,6 +133,7 @@ def test_timer_disable_explicit(self): (TIMER_DISABLED_LOG_LINE)) def _start_times_engine(self, cmd): + subprocess = concurrency.get_subprocess_module() process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, preexec_fn=os.setsid) self.add_process(process=process) diff --git a/st2reactor/tests/integration/test_sensor_container.py b/st2reactor/tests/integration/test_sensor_container.py index 018a825ac1..d3e7c04d2f 100644 --- a/st2reactor/tests/integration/test_sensor_container.py +++ b/st2reactor/tests/integration/test_sensor_container.py @@ -19,11 +19,10 @@ import signal import psutil -import eventlet -from eventlet.green import subprocess from oslo_config import cfg import st2tests.config +from st2common.util import concurrency from st2common.models.db import db_setup from st2reactor.container.process_container import PROCESS_EXIT_TIMEOUT from st2common.util.green.shell import run_command @@ -94,7 +93,7 @@ def test_child_processes_are_killed_on_sigint(self): process = self._start_sensor_container() # Give it some time to start up - eventlet.sleep(5) + concurrency.sleep(5) # Assert process has started and is running self.assertProcessIsRunning(process=process) @@ -110,7 +109,7 @@ def test_child_processes_are_killed_on_sigint(self): # SIGINT causes graceful shutdown so give it some time to gracefuly shut down the sensor # child processes - eventlet.sleep(PROCESS_EXIT_TIMEOUT + 1) + concurrency.sleep(PROCESS_EXIT_TIMEOUT + 1) # Verify parent and children processes have exited self.assertProcessExited(proc=pp) @@ -122,7 +121,7 @@ def test_child_processes_are_killed_on_sigterm(self): process = self._start_sensor_container() # Give it some time to start up - eventlet.sleep(3) + concurrency.sleep(3) # Verify container process and children sensor / wrapper processes are running pp = psutil.Process(process.pid) @@ -135,7 +134,7 @@ def test_child_processes_are_killed_on_sigterm(self): # SIGTERM causes graceful shutdown so give it some time to gracefuly shut down the sensor # child processes - eventlet.sleep(PROCESS_EXIT_TIMEOUT + 5) + concurrency.sleep(PROCESS_EXIT_TIMEOUT + 5) # Verify parent and children processes have exited self.assertProcessExited(proc=pp) @@ -147,7 +146,7 @@ def test_child_processes_are_killed_on_sigkill(self): process = self._start_sensor_container() # Give it some time to start up - eventlet.sleep(4) + concurrency.sleep(4) # Verify container process and children sensor / wrapper processes are running pp = psutil.Process(process.pid) @@ -159,7 +158,7 @@ def test_child_processes_are_killed_on_sigkill(self): process.send_signal(signal.SIGKILL) # Note: On SIGKILL processes should be killed instantly - eventlet.sleep(1) + concurrency.sleep(1) # Verify parent and children processes have exited self.assertProcessExited(proc=pp) @@ -175,7 +174,7 @@ def test_single_sensor_mode(self): pp = psutil.Process(process.pid) # Give it some time to start up - eventlet.sleep(4) + concurrency.sleep(4) stdout = process.stdout.read() self.assertTrue((b'--sensor-ref argument must be provided when running in single sensor ' @@ -191,7 +190,7 @@ def test_single_sensor_mode(self): pp = psutil.Process(process.pid) # Give it some time to start up - eventlet.sleep(8) + concurrency.sleep(8) # Container should exit and not respawn a sensor in single sensor mode stdout = process.stdout.read() @@ -200,12 +199,13 @@ def test_single_sensor_mode(self): self.assertTrue(b'Not respawning a sensor since running in single sensor mode') self.assertTrue(b'Process container quit with exit_code 110.') - eventlet.sleep(2) + concurrency.sleep(2) self.assertProcessExited(proc=pp) self.remove_process(process=process) def _start_sensor_container(self, cmd=DEFAULT_CMD): + subprocess = concurrency.get_subprocess_module() process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, preexec_fn=os.setsid) self.add_process(process=process) diff --git a/st2reactor/tests/integration/test_sensor_watcher.py b/st2reactor/tests/integration/test_sensor_watcher.py index 59ec452ce5..7a33cbb97e 100644 --- a/st2reactor/tests/integration/test_sensor_watcher.py +++ b/st2reactor/tests/integration/test_sensor_watcher.py @@ -13,10 +13,11 @@ # limitations under the License. from __future__ import absolute_import -import eventlet + from monotonic import monotonic from pyrabbit.api import Client +from st2common.util import concurrency from st2common.services.sensor_watcher import SensorWatcher from st2tests.base import IntegrationTestCase @@ -50,7 +51,7 @@ def delete_handler(sensor_db): start = monotonic() done = False while not done: - eventlet.sleep(0.01) + concurrency.sleep(0.01) sw_queues = self._get_sensor_watcher_amqp_queues(queue_name='st2.sensor.watch.covfefe') done = len(sw_queues) > 0 or ((monotonic() - start) < 5) diff --git a/st2reactor/tests/unit/test_process_container.py b/st2reactor/tests/unit/test_process_container.py index f353a0719b..f36b5277ea 100644 --- a/st2reactor/tests/unit/test_process_container.py +++ b/st2reactor/tests/unit/test_process_container.py @@ -16,11 +16,11 @@ import os import time -import eventlet from mock import (MagicMock, Mock, patch) import unittest2 from st2reactor.container.process_container import ProcessSensorContainer +from st2common.util import concurrency from st2common.models.db.pack import PackDB from st2common.persistence.pack import Pack @@ -35,8 +35,8 @@ class ProcessContainerTests(unittest2.TestCase): def test_no_sensors_dont_quit(self): process_container = ProcessSensorContainer(None, poll_interval=0.1) - process_container_thread = eventlet.spawn(process_container.run) - eventlet.sleep(0.5) + process_container_thread = concurrency.spawn(process_container.run) + concurrency.sleep(0.5) self.assertEqual(process_container.running(), 0) self.assertEqual(process_container.stopped(), False) process_container.shutdown() diff --git a/st2tests/st2tests/fixtures/generic/actions/action1.yaml b/st2tests/st2tests/fixtures/generic/actions/action1.yaml index 26522c26e3..15422c71b7 100644 --- a/st2tests/st2tests/fixtures/generic/actions/action1.yaml +++ b/st2tests/st2tests/fixtures/generic/actions/action1.yaml @@ -18,6 +18,9 @@ parameters: async_test: default: false type: boolean + action_secret: + type: string + secret: true runnerdummy: default: actiondummy immutable: true diff --git a/st2tests/st2tests/fixtures/generic/rules/rule1.yaml b/st2tests/st2tests/fixtures/generic/rules/rule1.yaml index a8c0809138..0e4302e7ca 100644 --- a/st2tests/st2tests/fixtures/generic/rules/rule1.yaml +++ b/st2tests/st2tests/fixtures/generic/rules/rule1.yaml @@ -3,6 +3,7 @@ action: parameters: ip1: '{{trigger.t1_p}}' ip2: '{{trigger}}' + action_secret: 'secret' ref: wolfpack.action-1 criteria: trigger.t1_p: diff --git a/st2tests/st2tests/resources/packs/pythonactions/actions/pascal_row.py b/st2tests/st2tests/resources/packs/pythonactions/actions/pascal_row.py index 6f42b63042..030670452b 100644 --- a/st2tests/st2tests/resources/packs/pythonactions/actions/pascal_row.py +++ b/st2tests/st2tests/resources/packs/pythonactions/actions/pascal_row.py @@ -36,6 +36,8 @@ def run(self, **kwargs): @staticmethod def _compute_pascal_row(row_index=0): + print('Pascal row action') + if row_index == 'a': return False, 'This is suppose to fail don\'t worry!!' elif row_index == 'b': diff --git a/st2tests/st2tests/resources/packs/pythonactions/actions/print_to_stdout_and_stderr.py b/st2tests/st2tests/resources/packs/pythonactions/actions/print_to_stdout_and_stderr.py new file mode 100644 index 0000000000..7c7a044748 --- /dev/null +++ b/st2tests/st2tests/resources/packs/pythonactions/actions/print_to_stdout_and_stderr.py @@ -0,0 +1,29 @@ +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import sys + +from st2common.runners.base_action import Action +from six.moves import range + + +class PrintToStdoutAndStderrAction(Action): + def run(self, stdout_count=3, stderr_count=3): + for index in range(0, stdout_count): + sys.stdout.write('stdout line %s\n' % (index)) + + for index in range(0, stderr_count): + sys.stderr.write('stderr line %s\n' % (index)) diff --git a/tools/launchdev.sh b/tools/launchdev.sh index 5cc6b13899..ad2e5d9dec 100755 --- a/tools/launchdev.sh +++ b/tools/launchdev.sh @@ -245,15 +245,18 @@ function st2start(){ screen -ls | grep mistral | cut -d. -f1 | awk '{print $1}' | xargs kill fi + # NOTE: We can't rely on latest version of screen with "-Logfile path" + # option so we need to use screen config file per screen window + # Run the st2 API server echo 'Starting screen session st2-api...' if [ "${use_gunicorn}" = true ]; then echo ' using gunicorn to run st2-api...' export ST2_CONFIG_PATH=${ST2_CONF} - screen -d -m -S st2-api ${VIRTUALENV}/bin/gunicorn \ + screen -L -c tools/screen-configs/st2api.conf -d -m -S st2-api ${VIRTUALENV}/bin/gunicorn \ st2api.wsgi:application -k eventlet -b "$BINDING_ADDRESS:9101" --workers 1 else - screen -d -m -S st2-api ${VIRTUALENV}/bin/python \ + screen -L -c tools/screen-configs/st2api.conf -d -m -S st2-api ${VIRTUALENV}/bin/python \ ./st2api/bin/st2api \ --config-file $ST2_CONF fi @@ -262,10 +265,10 @@ function st2start(){ if [ "${use_gunicorn}" = true ]; then echo ' using gunicorn to run st2-stream' export ST2_CONFIG_PATH=${ST2_CONF} - screen -d -m -S st2-stream ${VIRTUALENV}/bin/gunicorn \ + screen -L -c tools/screen-configs/st2stream.conf -d -m -S st2-stream ${VIRTUALENV}/bin/gunicorn \ st2stream.wsgi:application -k eventlet -b "$BINDING_ADDRESS:9102" --workers 1 else - screen -d -m -S st2-stream ${VIRTUALENV}/bin/python \ + screen -L -c tools/screen-configs/st2stream.conf -d -m -S st2-stream ${VIRTUALENV}/bin/python \ ./st2stream/bin/st2stream \ --config-file $ST2_CONF fi @@ -278,7 +281,7 @@ function st2start(){ WORKFLOW_ENGINE_NAME=st2-workflow-$i WORKFLOW_ENGINE_SCREENS+=($WORKFLOW_ENGINE_NAME) echo ' starting '$WORKFLOW_ENGINE_NAME'...' - screen -d -m -S $WORKFLOW_ENGINE_NAME ${VIRTUALENV}/bin/python \ + screen -L -c tools/screen-configs/st2workflowengine.conf -d -m -S $WORKFLOW_ENGINE_NAME ${VIRTUALENV}/bin/python \ ./st2actions/bin/st2workflowengine \ --config-file $ST2_CONF done @@ -291,14 +294,14 @@ function st2start(){ RUNNER_NAME=st2-actionrunner-$i RUNNER_SCREENS+=($RUNNER_NAME) echo ' starting '$RUNNER_NAME'...' - screen -d -m -S $RUNNER_NAME ${VIRTUALENV}/bin/python \ + screen -L -c tools/screen-configs/st2actionrunner.conf -d -m -S $RUNNER_NAME ${VIRTUALENV}/bin/python \ ./st2actions/bin/st2actionrunner \ --config-file $ST2_CONF done # Run the garbage collector service echo 'Starting screen session st2-garbagecollector' - screen -d -m -S st2-garbagecollector ${VIRTUALENV}/bin/python \ + screen -L -c tools/screen-configs/st2garbagecollector.conf -d -m -S st2-garbagecollector ${VIRTUALENV}/bin/python \ ./st2reactor/bin/st2garbagecollector \ --config-file $ST2_CONF @@ -310,38 +313,38 @@ function st2start(){ SCHEDULER_NAME=st2-scheduler-$i SCHEDULER_SCREENS+=($SCHEDULER_NAME) echo ' starting '$SCHEDULER_NAME'...' - screen -d -m -S $SCHEDULER_NAME ${VIRTUALENV}/bin/python \ + screen -L -c tools/screen-configs/st2scheduler.conf -d -m -S $SCHEDULER_NAME ${VIRTUALENV}/bin/python \ ./st2actions/bin/st2scheduler \ --config-file $ST2_CONF done # Run the sensor container server echo 'Starting screen session st2-sensorcontainer' - screen -d -m -S st2-sensorcontainer ${VIRTUALENV}/bin/python \ + screen -L -c tools/screen-configs/st2sensorcontainer.conf -d -m -S st2-sensorcontainer ${VIRTUALENV}/bin/python \ ./st2reactor/bin/st2sensorcontainer \ --config-file $ST2_CONF # Run the rules engine server echo 'Starting screen session st2-rulesengine...' - screen -d -m -S st2-rulesengine ${VIRTUALENV}/bin/python \ + screen -L -c tools/screen-configs/st2rulesengine.conf -d -m -S st2-rulesengine ${VIRTUALENV}/bin/python \ ./st2reactor/bin/st2rulesengine \ --config-file $ST2_CONF # Run the timer engine server echo 'Starting screen session st2-timersengine...' - screen -d -m -S st2-timersengine ${VIRTUALENV}/bin/python \ + screen -L -c tools/screen-configs/st2timersengine.conf -d -m -S st2-timersengine ${VIRTUALENV}/bin/python \ ./st2reactor/bin/st2timersengine \ --config-file $ST2_CONF # Run the results tracker echo 'Starting screen session st2-resultstracker...' - screen -d -m -S st2-resultstracker ${VIRTUALENV}/bin/python \ + screen -L -c tools/screen-configs/st2resultstracker.conf -d -m -S st2-resultstracker ${VIRTUALENV}/bin/python \ ./st2actions/bin/st2resultstracker \ --config-file $ST2_CONF # Run the actions notifier echo 'Starting screen session st2-notifier...' - screen -d -m -S st2-notifier ${VIRTUALENV}/bin/python \ + screen -L -c tools/screen-configs/st2notifier.conf -d -m -S st2-notifier ${VIRTUALENV}/bin/python \ ./st2actions/bin/st2notifier \ --config-file $ST2_CONF @@ -350,10 +353,10 @@ function st2start(){ if [ "${use_gunicorn}" = true ]; then echo ' using gunicorn to run st2-auth...' export ST2_CONFIG_PATH=${ST2_CONF} - screen -d -m -S st2-auth ${VIRTUALENV}/bin/gunicorn \ + screen -L -c tools/screen-configs/st2auth.conf -d -m -S st2-auth ${VIRTUALENV}/bin/gunicorn \ st2auth.wsgi:application -k eventlet -b "$BINDING_ADDRESS:9100" --workers 1 else - screen -d -m -S st2-auth ${VIRTUALENV}/bin/python \ + screen -L -c tools/screen-configs/st2auth.conf -d -m -S st2-auth ${VIRTUALENV}/bin/python \ ./st2auth/bin/st2auth \ --config-file $ST2_CONF fi @@ -364,18 +367,17 @@ function st2start(){ sudo mkdir -p $EXPORTS_DIR sudo chown -R ${CURRENT_USER}:${CURRENT_USER_GROUP} $EXPORTS_DIR echo 'Starting screen session st2-exporter...' - screen -d -m -S st2-exporter ${VIRTUALENV}/bin/python \ + screen -L -d -m -S st2-exporter ${VIRTUALENV}/bin/python \ ./st2exporter/bin/st2exporter \ --config-file $ST2_CONF fi if [ "${include_mistral}" = true ]; then - LOGDIR=${ST2_REPO}/logs # Run mistral-server echo 'Starting screen session mistral-server...' - screen -d -m -S mistral-server ${MISTRAL_REPO}/.venv/bin/python \ + screen -L -Logfile logs/screen-mistral-server.log -d -m -S mistral-server ${MISTRAL_REPO}/.venv/bin/python \ ${MISTRAL_REPO}/.venv/bin/mistral-server \ --server engine,executor \ --config-file $MISTRAL_CONF \ @@ -383,7 +385,7 @@ function st2start(){ # Run mistral-api echo 'Starting screen session mistral-api...' - screen -d -m -S mistral-api ${MISTRAL_REPO}/.venv/bin/python \ + screen -L -Logfile logs/screen-mistral-server.log -d -m -S mistral-api ${MISTRAL_REPO}/.venv/bin/python \ ${MISTRAL_REPO}/.venv/bin/mistral-server \ --server api \ --config-file $MISTRAL_CONF \ diff --git a/tools/screen-configs/st2actionrunner.conf b/tools/screen-configs/st2actionrunner.conf new file mode 100644 index 0000000000..c7c8d900a1 --- /dev/null +++ b/tools/screen-configs/st2actionrunner.conf @@ -0,0 +1,6 @@ +logfile logs/screen-st2actionrunner.log +logfile flush 1 +log on +logtstamp after 1 +logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" +logtstamp on diff --git a/tools/screen-configs/st2api.conf b/tools/screen-configs/st2api.conf new file mode 100644 index 0000000000..834313114b --- /dev/null +++ b/tools/screen-configs/st2api.conf @@ -0,0 +1,6 @@ +logfile logs/screen-st2api.log +logfile flush 1 +log on +logtstamp after 1 +logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" +logtstamp on diff --git a/tools/screen-configs/st2auth.conf b/tools/screen-configs/st2auth.conf new file mode 100644 index 0000000000..0bff41afcb --- /dev/null +++ b/tools/screen-configs/st2auth.conf @@ -0,0 +1,6 @@ +logfile logs/screen-st2auth.log +logfile flush 1 +log on +logtstamp after 1 +logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" +logtstamp on diff --git a/tools/screen-configs/st2garbagecollector.conf b/tools/screen-configs/st2garbagecollector.conf new file mode 100644 index 0000000000..dddb0f68f4 --- /dev/null +++ b/tools/screen-configs/st2garbagecollector.conf @@ -0,0 +1,6 @@ +logfile logs/screen-st2garbagecollector.log +logfile flush 1 +log on +logtstamp after 1 +logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" +logtstamp on diff --git a/tools/screen-configs/st2notifier.conf b/tools/screen-configs/st2notifier.conf new file mode 100644 index 0000000000..104d873a39 --- /dev/null +++ b/tools/screen-configs/st2notifier.conf @@ -0,0 +1,6 @@ +logfile logs/screen-st2notifier.log +logfile flush 1 +log on +logtstamp after 1 +logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" +logtstamp on diff --git a/tools/screen-configs/st2resultstracker.conf b/tools/screen-configs/st2resultstracker.conf new file mode 100644 index 0000000000..c5d454d4d0 --- /dev/null +++ b/tools/screen-configs/st2resultstracker.conf @@ -0,0 +1,6 @@ +logfile logs/screen-st2resultstracker.log +logfile flush 1 +log on +logtstamp after 1 +logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" +logtstamp on diff --git a/tools/screen-configs/st2rulesengine.conf b/tools/screen-configs/st2rulesengine.conf new file mode 100644 index 0000000000..1fca4d9807 --- /dev/null +++ b/tools/screen-configs/st2rulesengine.conf @@ -0,0 +1,6 @@ +logfile logs/screen-st2rulesengine.log +logfile flush 1 +log on +logtstamp after 1 +logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" +logtstamp on diff --git a/tools/screen-configs/st2scheduler.conf b/tools/screen-configs/st2scheduler.conf new file mode 100644 index 0000000000..1bbcbd81f6 --- /dev/null +++ b/tools/screen-configs/st2scheduler.conf @@ -0,0 +1,6 @@ +logfile logs/screen-st2scheduler.log +logfile flush 1 +log on +logtstamp after 1 +logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" +logtstamp on diff --git a/tools/screen-configs/st2sensorcontainer.conf b/tools/screen-configs/st2sensorcontainer.conf new file mode 100644 index 0000000000..735550d07b --- /dev/null +++ b/tools/screen-configs/st2sensorcontainer.conf @@ -0,0 +1,6 @@ +logfile logs/screen-st2sensorcontainer.log +logfile flush 1 +log on +logtstamp after 1 +logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" +logtstamp on diff --git a/tools/screen-configs/st2stream.conf b/tools/screen-configs/st2stream.conf new file mode 100644 index 0000000000..e6c6b8c09e --- /dev/null +++ b/tools/screen-configs/st2stream.conf @@ -0,0 +1,6 @@ +logfile logs/screen-st2stream.log +logfile flush 1 +log on +logtstamp after 1 +logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" +logtstamp on diff --git a/tools/screen-configs/st2timersengine.conf b/tools/screen-configs/st2timersengine.conf new file mode 100644 index 0000000000..5d358d8fec --- /dev/null +++ b/tools/screen-configs/st2timersengine.conf @@ -0,0 +1,6 @@ +logfile logs/screen-st2timersengine.log +logfile flush 1 +log on +logtstamp after 1 +logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" +logtstamp on diff --git a/tools/screen-configs/st2workflowengine.conf b/tools/screen-configs/st2workflowengine.conf new file mode 100644 index 0000000000..28d110e29c --- /dev/null +++ b/tools/screen-configs/st2workflowengine.conf @@ -0,0 +1,6 @@ +logfile logs/screen-st2workflowengine.log +logfile flush 1 +log on +logtstamp after 1 +logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\" +logtstamp on