diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6447c237ef..b503471209 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,27 @@ Changelog In development -------------- +Fixed +~~~~~ + +* Fix inadvertent regression in notifier service which would cause generic action trigger to only + be dispatched for completed states even if custom states were specified using + ``action_sensor.emit_when`` config option. (bug fix) + Reported by Shu Sugimoto (@shusugmt). #4591 +* Make sure we don't log auth token and api key inside st2api log file if those values are provided + via query parameter and not header (``?x-auth-token=foo``, ``?st2-api-key=bar``). (bug fix) #4592 + #4589 +* Fix rendering of ``{{ config_context. }}`` in orquesta task that references action from a + different pack (bug fix) #4570 #4567 +* Add missing default config location (``/etc/st2/st2.conf``) to the following services: + ``st2actionrunner``, ``st2scheduler``, ``st2workflowengine``. (bug fix) #4596 +* Update statsd metrics driver so any exception thrown by statsd library is treated as non fatal. + + Previously there was an edge case if user used a hostname instead of an IP address for metrics + backend server address. In such scenario, if hostname DNS resolution failed, statsd driver would + throw the exception which would propagate all the way up and break the application. (bug fix) #4597 + + Reported by Chris McKenzie. 2.10.3 - March 06, 2019 ----------------------- diff --git a/contrib/examples/actions/render_config_context.yaml b/contrib/examples/actions/render_config_context.yaml new file mode 100644 index 0000000000..f9c307f130 --- /dev/null +++ b/contrib/examples/actions/render_config_context.yaml @@ -0,0 +1,8 @@ +--- +name: render_config_context +pack: examples +description: Run render config context workflow +runner_type: orquesta +entry_point: workflows/render_config_context.yaml +enabled: true +parameters: {} diff --git a/contrib/examples/actions/workflows/render_config_context.yaml b/contrib/examples/actions/workflows/render_config_context.yaml new file mode 100644 index 0000000000..24f2d5394d --- /dev/null +++ b/contrib/examples/actions/workflows/render_config_context.yaml @@ -0,0 +1,7 @@ +version: 1.0 +description: Testing config context render". +tasks: + task1: + action: tests.render_config_context +output: + - context_value: <% task(task1).result.result.context_value %> diff --git a/st2actions/st2actions/config.py b/st2actions/st2actions/config.py index 4e03772e37..fbd661a59d 100644 --- a/st2actions/st2actions/config.py +++ b/st2actions/st2actions/config.py @@ -22,12 +22,14 @@ import st2common.config as common_config from st2common.constants.system import VERSION_STRING +from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH CONF = cfg.CONF def parse_args(args=None): - CONF(args=args, version=VERSION_STRING) + CONF(args=args, version=VERSION_STRING, + default_config_files=[DEFAULT_CONFIG_FILE_PATH]) def register_opts(): diff --git a/st2actions/st2actions/notifier/notifier.py b/st2actions/st2actions/notifier/notifier.py index b626bb5d13..9921167dc2 100644 --- a/st2actions/st2actions/notifier/notifier.py +++ b/st2actions/st2actions/notifier/notifier.py @@ -97,10 +97,7 @@ def process(self, execution_db): self._post_notify_triggers(liveaction_db=liveaction_db, execution_db=execution_db) - if cfg.CONF.action_sensor.enable: - with CounterWithTimer(key='notifier.generic_trigger.post'): - self._post_generic_trigger(liveaction_db=liveaction_db, - execution_db=execution_db) + self._post_generic_trigger(liveaction_db=liveaction_db, execution_db=execution_db) def _get_execution_for_liveaction(self, liveaction): execution = ActionExecution.get(liveaction__id=str(liveaction.id)) @@ -252,25 +249,26 @@ def _post_generic_trigger(self, liveaction_db=None, execution_db=None): LOG.debug(msg % (execution_id, execution_db.status, target_statuses), extra=extra) return - payload = {'execution_id': execution_id, - 'status': liveaction_db.status, - 'start_timestamp': str(liveaction_db.start_timestamp), - # deprecate 'action_name' at some point and switch to 'action_ref' - 'action_name': liveaction_db.action, - 'action_ref': liveaction_db.action, - 'runner_ref': self._get_runner_ref(liveaction_db.action), - 'parameters': liveaction_db.get_masked_parameters(), - 'result': liveaction_db.result} - # Use execution_id to extract trace rather than liveaction. execution_id - # will look-up an exact TraceDB while liveaction depending on context - # may not end up going to the DB. - trace_context = self._get_trace_context(execution_id=execution_id) - LOG.debug('POSTing %s for %s. Payload - %s. TraceContext - %s', - ACTION_TRIGGER_TYPE['name'], liveaction_db.id, payload, trace_context) - - with CounterWithTimer(key='notifier.generic_trigger.dispatch'): - self._trigger_dispatcher.dispatch(self._action_trigger, payload=payload, - trace_context=trace_context) + with CounterWithTimer(key='notifier.generic_trigger.post'): + payload = {'execution_id': execution_id, + 'status': liveaction_db.status, + 'start_timestamp': str(liveaction_db.start_timestamp), + # deprecate 'action_name' at some point and switch to 'action_ref' + 'action_name': liveaction_db.action, + 'action_ref': liveaction_db.action, + 'runner_ref': self._get_runner_ref(liveaction_db.action), + 'parameters': liveaction_db.get_masked_parameters(), + 'result': liveaction_db.result} + # Use execution_id to extract trace rather than liveaction. execution_id + # will look-up an exact TraceDB while liveaction depending on context + # may not end up going to the DB. + trace_context = self._get_trace_context(execution_id=execution_id) + LOG.debug('POSTing %s for %s. Payload - %s. TraceContext - %s', + ACTION_TRIGGER_TYPE['name'], liveaction_db.id, payload, trace_context) + + with CounterWithTimer(key='notifier.generic_trigger.dispatch'): + self._trigger_dispatcher.dispatch(self._action_trigger, payload=payload, + trace_context=trace_context) def _get_runner_ref(self, action_ref): """ diff --git a/st2actions/st2actions/scheduler/config.py b/st2actions/st2actions/scheduler/config.py index 27edfd6634..54addb77fb 100644 --- a/st2actions/st2actions/scheduler/config.py +++ b/st2actions/st2actions/scheduler/config.py @@ -19,6 +19,7 @@ from st2common import config as common_config from st2common.constants import system as sys_constants +from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH from st2common import log as logging @@ -26,7 +27,8 @@ def parse_args(args=None): - cfg.CONF(args=args, version=sys_constants.VERSION_STRING) + cfg.CONF(args=args, version=sys_constants.VERSION_STRING, + default_config_files=[DEFAULT_CONFIG_FILE_PATH]) def register_opts(): diff --git a/st2actions/st2actions/workflows/config.py b/st2actions/st2actions/workflows/config.py index ac659ef29c..b5fcfb9e23 100644 --- a/st2actions/st2actions/workflows/config.py +++ b/st2actions/st2actions/workflows/config.py @@ -19,10 +19,12 @@ from st2common import config as common_config from st2common.constants import system as sys_constants +from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH def parse_args(args=None): - cfg.CONF(args=args, version=sys_constants.VERSION_STRING) + cfg.CONF(args=args, version=sys_constants.VERSION_STRING, + default_config_files=[DEFAULT_CONFIG_FILE_PATH]) def register_opts(): diff --git a/st2actions/tests/unit/test_notifier.py b/st2actions/tests/unit/test_notifier.py index f82ecd0bf2..10246ef534 100644 --- a/st2actions/tests/unit/test_notifier.py +++ b/st2actions/tests/unit/test_notifier.py @@ -263,3 +263,83 @@ def test_post_generic_trigger_with_emit_condition(self, dispatch): payload=exp, trace_context={}) self.assertEqual(dispatch.call_count, 3) + + @mock.patch('oslo_config.cfg.CONF.action_sensor.enable', mock.MagicMock( + return_value=True)) + @mock.patch.object(Notifier, '_get_runner_ref', mock.MagicMock( + return_value='local-shell-cmd')) + @mock.patch.object(Notifier, '_get_trace_context', mock.MagicMock( + return_value={})) + @mock.patch('st2common.transport.reactor.TriggerDispatcher.dispatch') + @mock.patch('st2actions.notifier.notifier.LiveAction') + @mock.patch('st2actions.notifier.notifier.policy_service.apply_post_run_policies', mock.Mock()) + def test_process_post_generic_notify_trigger_on_completed_state_default(self, + mock_LiveAction, mock_dispatch): + # Verify that generic action trigger is posted on all completed states when action sensor + # is enabled + for status in LIVEACTION_STATUSES: + notifier = Notifier(connection=None, queues=[]) + + liveaction_db = LiveActionDB(id=bson.ObjectId(), action='core.local') + liveaction_db.status = status + execution = MOCK_EXECUTION + execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) + execution.status = liveaction_db.status + + mock_LiveAction.get_by_id.return_value = liveaction_db + + notifier = Notifier(connection=None, queues=[]) + notifier.process(execution) + + if status in LIVEACTION_COMPLETED_STATES: + exp = {'status': status, + 'start_timestamp': str(liveaction_db.start_timestamp), + 'result': {}, 'parameters': {}, + 'action_ref': u'core.local', + 'runner_ref': 'local-shell-cmd', + 'execution_id': str(MOCK_EXECUTION.id), + 'action_name': u'core.local'} + mock_dispatch.assert_called_with('core.st2.generic.actiontrigger', + payload=exp, trace_context={}) + + self.assertEqual(mock_dispatch.call_count, len(LIVEACTION_COMPLETED_STATES)) + + @mock.patch('oslo_config.cfg.CONF.action_sensor', mock.MagicMock( + enable=True, emit_when=['scheduled', 'pending', 'abandoned'])) + @mock.patch.object(Notifier, '_get_runner_ref', mock.MagicMock( + return_value='local-shell-cmd')) + @mock.patch.object(Notifier, '_get_trace_context', mock.MagicMock( + return_value={})) + @mock.patch('st2common.transport.reactor.TriggerDispatcher.dispatch') + @mock.patch('st2actions.notifier.notifier.LiveAction') + @mock.patch('st2actions.notifier.notifier.policy_service.apply_post_run_policies', mock.Mock()) + def test_process_post_generic_notify_trigger_on_custom_emit_when_states(self, + mock_LiveAction, mock_dispatch): + # Verify that generic action trigger is posted on all completed states when action sensor + # is enabled + for status in LIVEACTION_STATUSES: + notifier = Notifier(connection=None, queues=[]) + + liveaction_db = LiveActionDB(id=bson.ObjectId(), action='core.local') + liveaction_db.status = status + execution = MOCK_EXECUTION + execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) + execution.status = liveaction_db.status + + mock_LiveAction.get_by_id.return_value = liveaction_db + + notifier = Notifier(connection=None, queues=[]) + notifier.process(execution) + + if status in ['scheduled', 'pending', 'abandoned']: + exp = {'status': status, + 'start_timestamp': str(liveaction_db.start_timestamp), + 'result': {}, 'parameters': {}, + 'action_ref': u'core.local', + 'runner_ref': 'local-shell-cmd', + 'execution_id': str(MOCK_EXECUTION.id), + 'action_name': u'core.local'} + mock_dispatch.assert_called_with('core.st2.generic.actiontrigger', + payload=exp, trace_context={}) + + self.assertEqual(mock_dispatch.call_count, 3) diff --git a/st2common/st2common/metrics/drivers/statsd_driver.py b/st2common/st2common/metrics/drivers/statsd_driver.py index 163938f291..2324ab4b71 100644 --- a/st2common/st2common/metrics/drivers/statsd_driver.py +++ b/st2common/st2common/metrics/drivers/statsd_driver.py @@ -13,14 +13,28 @@ # See the License for the specific language governing permissions and # limitations under the License. +import socket +import logging as stdlib_logging from numbers import Number import statsd from oslo_config import cfg +from st2common import log as logging from st2common.metrics.base import BaseMetricsDriver from st2common.metrics.utils import check_key from st2common.metrics.utils import get_full_key_name +from st2common.util.misc import ignore_and_log_exception + + +LOG = logging.getLogger(__name__) + +# Which exceptions thrown by statsd library should be considered as non-fatal +NON_FATAL_EXC_CLASSES = [ + socket.error, + IOError, + OSError +] __all__ = [ 'StatsdDriver' @@ -30,11 +44,22 @@ class StatsdDriver(BaseMetricsDriver): """ StatsD Implementation of the metrics driver + + NOTE: Statsd uses UDP which is "fire and forget" and any kind of send error is not fatal. There + is an issue with python-statsd library though which doesn't ignore DNS resolution related errors + and bubbles them all the way up. + + This of course breaks the application. Any kind of metric related errors should be considered + as non-fatal and not degrade application in any way if an error occurs. That's why we wrap all + the statsd library calls here to ignore the errors and just log them. """ + def __init__(self): statsd.Connection.set_defaults(host=cfg.CONF.metrics.host, port=cfg.CONF.metrics.port, sample_rate=cfg.CONF.metrics.sample_rate) + @ignore_and_log_exception(exc_classes=NON_FATAL_EXC_CLASSES, logger=LOG, + level=stdlib_logging.WARNING) def time(self, key, time): """ Timer metric @@ -46,6 +71,8 @@ def time(self, key, time): timer = statsd.Timer('') timer.send(key, time) + @ignore_and_log_exception(exc_classes=NON_FATAL_EXC_CLASSES, logger=LOG, + level=stdlib_logging.WARNING) def inc_counter(self, key, amount=1): """ Increment counter @@ -57,6 +84,8 @@ def inc_counter(self, key, amount=1): counter = statsd.Counter(key) counter.increment(delta=amount) + @ignore_and_log_exception(exc_classes=NON_FATAL_EXC_CLASSES, logger=LOG, + level=stdlib_logging.WARNING) def dec_counter(self, key, amount=1): """ Decrement metric @@ -68,6 +97,8 @@ def dec_counter(self, key, amount=1): counter = statsd.Counter(key) counter.decrement(delta=amount) + @ignore_and_log_exception(exc_classes=NON_FATAL_EXC_CLASSES, logger=LOG, + level=stdlib_logging.WARNING) def set_gauge(self, key, value): """ Set gauge value. @@ -79,6 +110,8 @@ def set_gauge(self, key, value): gauge = statsd.Gauge(key) gauge.send(None, value) + @ignore_and_log_exception(exc_classes=NON_FATAL_EXC_CLASSES, logger=LOG, + level=stdlib_logging.WARNING) def inc_gauge(self, key, amount=1): """ Increment gauge value. @@ -90,6 +123,8 @@ def inc_gauge(self, key, amount=1): gauge = statsd.Gauge(key) gauge.increment(None, amount) + @ignore_and_log_exception(exc_classes=NON_FATAL_EXC_CLASSES, logger=LOG, + level=stdlib_logging.WARNING) def dec_gauge(self, key, amount=1): """ Decrement gauge value. diff --git a/st2common/st2common/middleware/logging.py b/st2common/st2common/middleware/logging.py index e4d30a1e7d..9eeaa4cfeb 100644 --- a/st2common/st2common/middleware/logging.py +++ b/st2common/st2common/middleware/logging.py @@ -14,16 +14,28 @@ # limitations under the License. from __future__ import absolute_import + import time import types import itertools +from oslo_config import cfg + from st2common.constants.api import REQUEST_ID_HEADER +from st2common.constants.auth import QUERY_PARAM_ATTRIBUTE_NAME +from st2common.constants.auth import QUERY_PARAM_API_KEY_ATTRIBUTE_NAME +from st2common.constants.secrets import MASKED_ATTRIBUTE_VALUE +from st2common.constants.secrets import MASKED_ATTRIBUTES_BLACKLIST from st2common import log as logging from st2common.router import Request, NotFoundException LOG = logging.getLogger(__name__) +SECRET_QUERY_PARAMS = [ + QUERY_PARAM_ATTRIBUTE_NAME, + QUERY_PARAM_API_KEY_ATTRIBUTE_NAME +] + MASKED_ATTRIBUTES_BLACKLIST + try: clock = time.perf_counter except AttributeError: @@ -46,12 +58,20 @@ def __call__(self, environ, start_response): request = Request(environ) + query_params = request.GET.dict_of_lists() + + # Mask secret / sensitive query params + secret_query_params = SECRET_QUERY_PARAMS + cfg.CONF.log.mask_secrets_blacklist + for param_name in secret_query_params: + if param_name in query_params: + query_params[param_name] = MASKED_ATTRIBUTE_VALUE + # Log the incoming request values = { 'method': request.method, 'path': request.path, 'remote_addr': request.remote_addr, - 'query': request.GET.dict_of_lists(), + 'query': query_params, 'request_id': request.headers.get(REQUEST_ID_HEADER, None) } diff --git a/st2common/st2common/services/workflows.py b/st2common/st2common/services/workflows.py index 65a2a21040..db5bc3c6f0 100644 --- a/st2common/st2common/services/workflows.py +++ b/st2common/st2common/services/workflows.py @@ -535,9 +535,12 @@ def request_action_execution(wf_ex_db, task_ex_db, st2_ctx, ac_ex_req, delay=Non # Identify the runner for the action. runner_type_db = action_utils.get_runnertype_by_name(action_db.runner_type['name']) + # Identify action pack name + pack_name = action_ref.split('.')[0] if action_ref else st2_ctx.get('pack') + # Set context for the action execution. ac_ex_ctx = { - 'pack': st2_ctx.get('pack'), + 'pack': pack_name, 'user': st2_ctx.get('user'), 'parent': st2_ctx, 'orquesta': { @@ -545,7 +548,7 @@ def request_action_execution(wf_ex_db, task_ex_db, st2_ctx, ac_ex_req, delay=Non 'task_execution_id': str(task_ex_db.id), 'task_name': task_ex_db.task_name, 'task_id': task_ex_db.task_id - } + }, } if st2_ctx.get('api_user'): diff --git a/st2common/st2common/util/misc.py b/st2common/st2common/util/misc.py index 0768e2cb35..02d8f31513 100644 --- a/st2common/st2common/util/misc.py +++ b/st2common/st2common/util/misc.py @@ -15,10 +15,13 @@ from __future__ import absolute_import +import logging + import os import re import sys import collections +import functools import six @@ -26,7 +29,8 @@ 'prefix_dict_keys', 'compare_path_file_name', 'lowercase_value', - 'get_field_name_from_mongoengine_error' + 'get_field_name_from_mongoengine_error', + ] @@ -168,3 +172,28 @@ def get_field_name_from_mongoengine_error(exc): return match.groups()[0] return msg + + +def ignore_and_log_exception(exc_classes=(Exception,), logger=None, level=logging.WARNING): + """ + Decorator which catches the provided exception classes and logs them instead of letting them + bubble all the way up. + """ + exc_classes = tuple(exc_classes) + + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except exc_classes as e: + if len(args) >= 1 and getattr(args[0], '__class__', None): + func_name = '%s.%s' % (args[0].__class__.__name__, func.__name__) + else: + func_name = func.__name__ + + message = ('Exception in fuction "%s": %s' % (func_name, str(e))) + logger.log(level, message) + + return wrapper + return decorator diff --git a/st2common/tests/unit/services/test_workflow.py b/st2common/tests/unit/services/test_workflow.py index dc2354a9fb..735b56788c 100644 --- a/st2common/tests/unit/services/test_workflow.py +++ b/st2common/tests/unit/services/test_workflow.py @@ -28,7 +28,9 @@ from st2common.exceptions import action as action_exc from st2common.models.db import liveaction as lv_db_models from st2common.models.db import execution as ex_db_models +from st2common.models.db import pack as pk_db_models from st2common.persistence import execution as ex_db_access +from st2common.persistence import pack as pk_db_access from st2common.persistence import workflow as wf_db_access from st2common.services import action as action_service from st2common.services import workflows as workflow_service @@ -40,8 +42,12 @@ TEST_PACK = 'orquesta_tests' TEST_PACK_PATH = st2tests.fixturesloader.get_fixtures_packs_base_path() + '/' + TEST_PACK +PACK_7 = 'dummy_pack_7' +PACK_7_PATH = st2tests.fixturesloader.get_fixtures_packs_base_path() + '/' + PACK_7 + PACKS = [ TEST_PACK_PATH, + PACK_7_PATH, st2tests.fixturesloader.get_fixtures_packs_base_path() + '/core' ] @@ -346,3 +352,67 @@ def test_evaluate_action_execution_delay(self): ac_ex_req = {'action': 'core.noop', 'input': None, 'item_id': 1} actual_delay = workflow_service.eval_action_execution_delay(task_ex_req, ac_ex_req, True) self.assertIsNone(actual_delay) + + def test_request_action_execution_render(self): + # Manually create ConfigDB + output = 'Testing' + value = { + "config_item_one": output + } + config_db = pk_db_models.ConfigDB(pack=PACK_7, values=value) + config = pk_db_access.Config.add_or_update(config_db) + self.assertEqual(len(config), 3) + + wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, 'render_config_context.yaml') + + # Manually create the liveaction and action execution objects without publishing. + lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name']) + lv_ac_db, ac_ex_db = action_service.create_request(lv_ac_db) + + # Request the workflow execution. + wf_def = self.get_wf_def(TEST_PACK_PATH, wf_meta) + st2_ctx = self.mock_st2_context(ac_ex_db) + wf_ex_db = workflow_service.request(wf_def, ac_ex_db, st2_ctx) + spec_module = specs_loader.get_spec_module(wf_ex_db.spec['catalog']) + wf_spec = spec_module.WorkflowSpec.deserialize(wf_ex_db.spec) + + # Pass down appropriate st2 context to the task and action execution(s). + root_st2_ctx = wf_ex_db.context.get('st2', {}) + st2_ctx = { + 'execution_id': wf_ex_db.action_execution, + 'user': root_st2_ctx.get('user'), + 'pack': root_st2_ctx.get('pack') + } + + # Manually request task execution. + task_route = 0 + task_id = 'task1' + task_spec = wf_spec.tasks.get_task(task_id) + task_ctx = {'foo': 'bar'} + + task_ex_req = { + 'id': task_id, + 'route': task_route, + 'spec': task_spec, + 'ctx': task_ctx, + 'actions': [ + {'action': 'dummy_pack_7.render_config_context', 'input': None} + ] + } + workflow_service.request_task_execution(wf_ex_db, st2_ctx, task_ex_req) + + # Check task execution is saved to the database. + task_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id)) + self.assertEqual(len(task_ex_dbs), 1) + workflow_service.request_task_execution(wf_ex_db, st2_ctx, task_ex_req) + + # Manually request action execution + task_ex_db = task_ex_dbs[0] + action_ex_db = workflow_service.request_action_execution(wf_ex_db, task_ex_db, st2_ctx, + task_ex_req['actions'][0]) + + # Check required attributes. + self.assertIsNotNone(str(action_ex_db.id)) + self.assertEqual(task_ex_db.workflow_execution, str(wf_ex_db.id)) + expected_parameters = {'value1': output} + self.assertEqual(expected_parameters, action_ex_db.parameters) diff --git a/st2common/tests/unit/test_logging_middleware.py b/st2common/tests/unit/test_logging_middleware.py new file mode 100644 index 0000000000..014634e414 --- /dev/null +++ b/st2common/tests/unit/test_logging_middleware.py @@ -0,0 +1,74 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import unittest2 + +from oslo_config import cfg + +from st2common.middleware.logging import LoggingMiddleware +from st2common.constants.secrets import MASKED_ATTRIBUTE_VALUE + +__all__ = [ + 'LoggingMiddlewareTestCase' +] + + +class LoggingMiddlewareTestCase(unittest2.TestCase): + @mock.patch('st2common.middleware.logging.LOG') + @mock.patch('st2common.middleware.logging.Request') + def test_secret_parameters_are_masked_in_log_message(self, mock_request, mock_log): + + def app(environ, custom_start_response): + custom_start_response(status='200 OK', headers=[('Content-Length', 100)]) + return [None] + + router = mock.Mock() + endpoint = mock.Mock() + router.match.return_value = (endpoint, None) + middleware = LoggingMiddleware(app=app, router=router) + + cfg.CONF.set_override(group='log', name='mask_secrets_blacklist', + override=['blacklisted_4', 'blacklisted_5']) + + environ = {} + mock_request.return_value.GET.dict_of_lists.return_value = { + 'foo': 'bar', + 'bar': 'baz', + 'x-auth-token': 'secret', + 'st2-api-key': 'secret', + 'password': 'secret', + 'st2_auth_token': 'secret', + 'token': 'secret', + 'blacklisted_4': 'super secret', + 'blacklisted_5': 'super secret', + } + middleware(environ=environ, start_response=mock.Mock()) + + expected_query = { + 'foo': 'bar', + 'bar': 'baz', + 'x-auth-token': MASKED_ATTRIBUTE_VALUE, + 'st2-api-key': MASKED_ATTRIBUTE_VALUE, + 'password': MASKED_ATTRIBUTE_VALUE, + 'token': MASKED_ATTRIBUTE_VALUE, + 'st2_auth_token': MASKED_ATTRIBUTE_VALUE, + 'blacklisted_4': MASKED_ATTRIBUTE_VALUE, + 'blacklisted_5': MASKED_ATTRIBUTE_VALUE, + } + + call_kwargs = mock_log.info.call_args_list[0][1] + query = call_kwargs['extra']['query'] + self.assertEqual(query, expected_query) diff --git a/st2common/tests/unit/test_metrics.py b/st2common/tests/unit/test_metrics.py index dbd2750432..1c7543ca7b 100644 --- a/st2common/tests/unit/test_metrics.py +++ b/st2common/tests/unit/test_metrics.py @@ -12,9 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime, timedelta + +import socket +from datetime import datetime +from datetime import timedelta import unittest2 +import mock from mock import patch, MagicMock from oslo_config import cfg @@ -200,6 +204,52 @@ def test_get_full_key_name(self): result = get_full_key_name('api.requests') self.assertEqual(result, 'st2.prod.api.requests') + @patch('st2common.metrics.drivers.statsd_driver.LOG') + @patch('st2common.metrics.drivers.statsd_driver.statsd') + def test_driver_socket_exceptions_are_not_fatal(self, statsd, mock_log): + # Socket errors such as DNS resolution errors should be considered non fatal and ignored + mock_logger = mock.Mock() + StatsdDriver.logger = mock_logger + + # 1. timer + mock_timer = MagicMock(side_effect=socket.error('error 1')) + statsd.Timer('').send.side_effect = mock_timer + params = ('test', 10) + self._driver.time(*params) + statsd.Timer('').send.assert_called_with('st2.test', 10) + + # 2. counter + key = 'test' + mock_counter = MagicMock(side_effect=socket.error('error 2')) + statsd.Counter(key).increment.side_effect = mock_counter + self._driver.inc_counter(key) + mock_counter.assert_called_once_with(delta=1) + + key = 'test' + mock_counter = MagicMock(side_effect=socket.error('error 3')) + statsd.Counter(key).decrement.side_effect = mock_counter + self._driver.dec_counter(key) + mock_counter.assert_called_once_with(delta=1) + + # 3. gauge + params = ('key', 100) + mock_gauge = MagicMock(side_effect=socket.error('error 4')) + statsd.Gauge().send.side_effect = mock_gauge + self._driver.set_gauge(*params) + mock_gauge.assert_called_once_with(None, params[1]) + + params = ('key1',) + mock_gauge = MagicMock(side_effect=socket.error('error 5')) + statsd.Gauge().increment.side_effect = mock_gauge + self._driver.inc_gauge(*params) + mock_gauge.assert_called_once_with(None, 1) + + params = ('key1',) + mock_gauge = MagicMock(side_effect=socket.error('error 6')) + statsd.Gauge().decrement.side_effect = mock_gauge + self._driver.dec_gauge(*params) + mock_gauge.assert_called_once_with(None, 1) + class TestCounterContextManager(unittest2.TestCase): @patch('st2common.metrics.base.METRICS') diff --git a/st2tests/integration/orquesta/test_wiring.py b/st2tests/integration/orquesta/test_wiring.py index 0306897810..e62e957377 100644 --- a/st2tests/integration/orquesta/test_wiring.py +++ b/st2tests/integration/orquesta/test_wiring.py @@ -144,3 +144,16 @@ def test_output_on_error(self): self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) self.assertDictEqual(ex.result, expected_result) + + def test_config_context_renders(self): + config_value = "Testing" + wf_name = 'examples.render_config_context' + + expected_output = {'context_value': config_value} + expected_result = {'output': expected_output} + + ex = self._execute_workflow(wf_name) + ex = self._wait_for_completion(ex) + + self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) + self.assertDictEqual(ex.result, expected_result) diff --git a/st2tests/st2tests/base.py b/st2tests/st2tests/base.py index b0c7e8eddb..3fdc91816b 100644 --- a/st2tests/st2tests/base.py +++ b/st2tests/st2tests/base.py @@ -618,7 +618,8 @@ def mock_st2_context(self, ac_ex_db, context=None): st2_ctx = { 'st2': { 'api_url': api_util.get_full_public_api_url(), - 'action_execution_id': str(ac_ex_db.id) + 'action_execution_id': str(ac_ex_db.id), + 'user': 'stanley' } } diff --git a/st2tests/st2tests/fixtures/packs/configs/dummy_pack_7.yaml b/st2tests/st2tests/fixtures/packs/configs/dummy_pack_7.yaml new file mode 100644 index 0000000000..6b01d06715 --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/configs/dummy_pack_7.yaml @@ -0,0 +1,2 @@ +--- +config_item_one: "testing" diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render_config_context.py b/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render_config_context.py new file mode 100644 index 0000000000..97ab48fea0 --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render_config_context.py @@ -0,0 +1,7 @@ +from st2common.runners.base_action import Action + + +class PrintPythonVersionAction(Action): + + def run(self, value1): + return {"context_value": value1} diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render_config_context.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render_config_context.yaml new file mode 100644 index 0000000000..e67d323c65 --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_7/actions/render_config_context.yaml @@ -0,0 +1,12 @@ +--- +name: render_config_context +runner_type: python-script +description: Action that uses config context +enabled: true +entry_point: render_config_context.py +parameters: + value1: + description: Input for render_config_context. Defaults to config_context value. + required: false + type: "string" + default: "{{ config_context.config_item_one }}" diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_7/config.schema.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_7/config.schema.yaml new file mode 100644 index 0000000000..731f42d98c --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_7/config.schema.yaml @@ -0,0 +1,5 @@ +--- +config_item_one: + description: "Item use to test config context." + type: "string" + required: true diff --git a/st2tests/st2tests/fixtures/packs/orquesta_tests/actions/render_config_context.yaml b/st2tests/st2tests/fixtures/packs/orquesta_tests/actions/render_config_context.yaml new file mode 100644 index 0000000000..2f720cfd25 --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/orquesta_tests/actions/render_config_context.yaml @@ -0,0 +1,8 @@ +--- +name: render_config_context +pack: orquesta_tests +description: Run render config context workflow +runner_type: orquesta +entry_point: workflows/render_config_context.yaml +enabled: true +parameters: {} diff --git a/st2tests/st2tests/fixtures/packs/orquesta_tests/actions/workflows/render_config_context.yaml b/st2tests/st2tests/fixtures/packs/orquesta_tests/actions/workflows/render_config_context.yaml new file mode 100644 index 0000000000..5683cbe98f --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/orquesta_tests/actions/workflows/render_config_context.yaml @@ -0,0 +1,7 @@ +version: 1.0 +description: Testing config context render". +tasks: + task1: + action: dummy_pack_7.render_config_context +output: + - context_value: <% task(task1).result.result.context_value %> diff --git a/tools/launchdev.sh b/tools/launchdev.sh index 764bcc129c..63debe2466 100755 --- a/tools/launchdev.sh +++ b/tools/launchdev.sh @@ -7,7 +7,7 @@ function usage() { subcommand=$1; shift runner_count=1 use_gunicorn=true -copy_examples=false +copy_test_packs=false load_content=true use_ipv6=false include_mistral=false @@ -21,7 +21,7 @@ while getopts ":r:gxcu6m" o; do use_gunicorn=false ;; x) - copy_examples=true + copy_test_packs=true ;; c) load_content=false @@ -201,9 +201,20 @@ function st2start(){ cp -Rp ./contrib/core/ $PACKS_BASE_DIR cp -Rp ./contrib/packs/ $PACKS_BASE_DIR - if [ "$copy_examples" = true ]; then - echo "Copying examples from ./contrib/examples to $PACKS_BASE_DIR" + if [ "$copy_test_packs" = true ]; then + echo "Copying test packs examples and tests to $PACKS_BASE_DIR" cp -Rp ./contrib/examples $PACKS_BASE_DIR + # Clone st2tests in /tmp directory. + pushd /tmp + git clone https://github.com/StackStorm/st2tests.git + ret=$? + if [ ${ret} -eq 0 ]; then + cp -Rp ./st2tests/packs/tests $PACKS_BASE_DIR + rm -R st2tests/ + else + echo "Failed to clone st2tests repo" + fi + popd fi # activate virtualenv to set PYTHONPATH @@ -391,6 +402,13 @@ function st2start(){ --config-file $ST2_CONF --register-all fi + if [ "$copy_test_packs" = true ]; then + st2 run packs.setup_virtualenv packs=tests + if [ $? != 0 ]; then + echo "Warning: Unable to setup virtualenv for the \"tests\" pack. Please setup virtualenv for the \"tests\" pack before running integration tests" + fi + fi + # List screen sessions screen -ls || exit 0 }