From 70b8de57dc6aabdeeecc619f07bdaa6f6d7e7b75 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 6 Feb 2019 10:51:26 +0100 Subject: [PATCH 1/6] Update service_setup service entry / bootstrap code so we exclude log messages with log level "AUDIT" if log level is set to INFO or higher. This way we avoid issues with duplicate AUDIT messages in production deployments. In production deployments default log level is set to INFO which means that all service log files will also contain AUDIT log messages because AUDIT level if the highest. This is not desired, because we already log AUDIT log messages in a dedicated AUDIT log file. NOTE: Audit messages will still go in the service log file is log level is set to DEBUG (that's desired during debugging). --- st2common/st2common/log.py | 3 +++ st2common/st2common/service_setup.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/st2common/st2common/log.py b/st2common/st2common/log.py index 1b9186fac3..46efd03bb9 100644 --- a/st2common/st2common/log.py +++ b/st2common/st2common/log.py @@ -46,6 +46,9 @@ 'ignore_statsd_log_messages' ] +# NOTE: We set AUDIT to the highest log level which means AUDIT log messages will always be +# included (e.g. also if log level is set to INFO). To avoid that, we need to explicitly filter +# out AUDIT log level in service setup code. logging.AUDIT = logging.CRITICAL + 10 logging.addLevelName(logging.AUDIT, 'AUDIT') diff --git a/st2common/st2common/service_setup.py b/st2common/st2common/service_setup.py index 61fbea6bce..bf82e00192 100644 --- a/st2common/st2common/service_setup.py +++ b/st2common/st2common/service_setup.py @@ -22,6 +22,7 @@ import os import sys import traceback +import logging as stdlib_logging from oslo_config import cfg @@ -35,6 +36,7 @@ from st2common.models.utils.profiling import enable_profiling from st2common import triggers from st2common.rbac.migrations import run_all as run_all_rbac_migrations +from st2common.logging.filters import LogLevelFilter # Note: This is here for backward compatibility. # Function has been moved in a standalone module to avoid expensive in-direct @@ -112,6 +114,20 @@ def setup(service, config, setup_db=True, register_mq_exchanges=True, else: raise e + exclude_log_levels = [stdlib_logging.AUDIT] + handlers = stdlib_logging.getLoggerClass().manager.root.handlers + + for handler in handlers: + # If log level is not set to DEBUG we filter out "AUDIT" log messages. This way we avoid + # duplicate "AUDIT" messages in production deployments where default service log level is + # set to "INFO" and we already log messages with level AUDIT to a special dedicated log + # file. + ignore_audit_log_messages = (handler.level >= stdlib_logging.INFO and + handler.level < stdlib_logging.AUDIT) + if not is_debug_enabled and ignore_audit_log_messages: + LOG.debug('Excluding log messages with level "AUDIT" for handler "%s"' % (handler)) + handler.addFilter(LogLevelFilter(log_levels=exclude_log_levels)) + if not is_debug_enabled: # NOTE: statsd logger logs everything by default under INFO so we ignore those log # messages unless verbose / debug mode is used From 560250833824e8efc11b43553fdd0e29aaba1abb Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 6 Feb 2019 10:58:37 +0100 Subject: [PATCH 2/6] Add changelog entry. --- CHANGELOG.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9d2aee0cd8..d9cd946224 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -21,7 +21,14 @@ Added Changed ~~~~~~~ -* Changed the ``inquiries`` API path from ``/exp`` to ``/api/v1`` #4495 +* Changed the ``inquiries`` API path from ``/exp`` to ``/api/v1``. #4495 +* Update logging code so we exclude log messages with log level ``AUDIT`` from a default service + log file. Log messages with level ``AUDIT`` are already logged in a dedicated + ``.audit.log`` file so there is no need for them to be duplicated in regular service log + file. + + NOTE: To aid with debugging, audit log messages are also included in a regular log file when log + level is set to DEBUG. (improvement) Fixed ~~~~~ From 09feb5d6e15d68b5104bda27050a8a70ec9dcd75 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 6 Feb 2019 11:13:00 +0100 Subject: [PATCH 3/6] Update changelog entry. --- CHANGELOG.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d9cd946224..c4be32dd9d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -23,12 +23,14 @@ Changed * Changed the ``inquiries`` API path from ``/exp`` to ``/api/v1``. #4495 * Update logging code so we exclude log messages with log level ``AUDIT`` from a default service - log file. Log messages with level ``AUDIT`` are already logged in a dedicated - ``.audit.log`` file so there is no need for them to be duplicated in regular service log - file. + log file (e.g. ``st2api.log``). Log messages with level ``AUDIT`` are already logged in a + dedicated service audit log file (e.g. ``st2api.audit.log``) so there is no need for them to also + be duplicated and included in regular service log file. NOTE: To aid with debugging, audit log messages are also included in a regular log file when log - level is set to DEBUG. (improvement) + level is set to ``DEBUG`` or ``system.debug`` config option is set to ``True``. + + Reported by Nick Maludy. (improvement) #4538 #4502 Fixed ~~~~~ From 11111ffd5f126e252537114c5ce550f64d783659 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 6 Feb 2019 13:30:23 +0100 Subject: [PATCH 4/6] Add an integration test case which verifies that service setup log level based filtering works correctly. --- .../test_service_setup_log_level_filtering.py | 123 ++++++++++++++++++ .../fixtures/conf/logging.api.audit.conf | 44 +++++++ .../fixtures/conf/logging.api.debug.conf | 44 +++++++ .../fixtures/conf/logging.api.info.conf | 44 +++++++ .../conf/st2.tests.api.audit_log_level.conf | 99 ++++++++++++++ .../conf/st2.tests.api.debug_log_level.conf | 99 ++++++++++++++ .../conf/st2.tests.api.info_log_level.conf | 99 ++++++++++++++ .../conf/st2.tests.api.system_debug_true.conf | 99 ++++++++++++++ .../st2tests/fixtures/conf/st2.tests.conf | 100 ++++++++++++++ 9 files changed, 751 insertions(+) create mode 100644 st2common/tests/integration/test_service_setup_log_level_filtering.py create mode 100644 st2tests/st2tests/fixtures/conf/logging.api.audit.conf create mode 100644 st2tests/st2tests/fixtures/conf/logging.api.debug.conf create mode 100644 st2tests/st2tests/fixtures/conf/logging.api.info.conf create mode 100644 st2tests/st2tests/fixtures/conf/st2.tests.api.audit_log_level.conf create mode 100644 st2tests/st2tests/fixtures/conf/st2.tests.api.debug_log_level.conf create mode 100644 st2tests/st2tests/fixtures/conf/st2.tests.api.info_log_level.conf create mode 100644 st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true.conf create mode 100644 st2tests/st2tests/fixtures/conf/st2.tests.conf diff --git a/st2common/tests/integration/test_service_setup_log_level_filtering.py b/st2common/tests/integration/test_service_setup_log_level_filtering.py new file mode 100644 index 0000000000..077d4a9f38 --- /dev/null +++ b/st2common/tests/integration/test_service_setup_log_level_filtering.py @@ -0,0 +1,123 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os +import sys +import signal + +import eventlet +from eventlet.green import subprocess + +from st2tests.base import IntegrationTestCase +from st2tests.fixturesloader import get_fixtures_base_path + +__all__ = [ + 'ServiceSetupLogLevelFilteringTestCase' +] + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + +FIXTURES_DIR = get_fixtures_base_path() + +ST2_CONFIG_INFO_LL_PATH = os.path.join(FIXTURES_DIR, 'conf/st2.tests.api.info_log_level.conf') +ST2_CONFIG_INFO_LL_PATH = os.path.abspath(ST2_CONFIG_INFO_LL_PATH) + +ST2_CONFIG_DEBUG_LL_PATH = os.path.join(FIXTURES_DIR, 'conf/st2.tests.api.debug_log_level.conf') +ST2_CONFIG_DEBUG_LL_PATH = os.path.abspath(ST2_CONFIG_DEBUG_LL_PATH) + +ST2_CONFIG_AUDIT_LL_PATH = os.path.join(FIXTURES_DIR, 'conf/st2.tests.api.audit_log_level.conf') +ST2_CONFIG_AUDIT_LL_PATH = os.path.abspath(ST2_CONFIG_AUDIT_LL_PATH) + +ST2_CONFIG_SYSTEM_DEBUG_PATH = os.path.join(FIXTURES_DIR, + 'conf/st2.tests.api.system_debug_true.conf') +ST2_CONFIG_SYSTEM_DEBUG_PATH = os.path.abspath(ST2_CONFIG_SYSTEM_DEBUG_PATH) + +PYTHON_BINARY = sys.executable + +ST2API_BINARY = os.path.join(BASE_DIR, '../../../st2api/bin/st2api') +ST2API_BINARY = os.path.abspath(ST2API_BINARY) + +CMD = [PYTHON_BINARY, ST2API_BINARY, '--config-file'] + + +class ServiceSetupLogLevelFilteringTestCase(IntegrationTestCase): + def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): + # 1. INFO log level - audit messages should not be included + process = self._start_process(config_path=ST2_CONFIG_INFO_LL_PATH) + self.add_process(process=process) + + # Give it some time to start up + eventlet.sleep(3) + process.send_signal(signal.SIGKILL) + + # First 3 log lines are debug messages about the environment which are always logged + stdout = '\n'.join(process.stdout.read().split('\n')[3:]) + + self.assertTrue('INFO [-]' in stdout) + self.assertTrue('DEBUG [-]' not in stdout) + self.assertTrue('AUDIT [-]' not in stdout) + + # 2. DEBUG log level - audit messages should be included + process = self._start_process(config_path=ST2_CONFIG_DEBUG_LL_PATH) + self.add_process(process=process) + + # Give it some time to start up + eventlet.sleep(3) + process.send_signal(signal.SIGKILL) + + # First 3 log lines are debug messages about the environment which are always logged + stdout = '\n'.join(process.stdout.read().split('\n')[3:]) + + self.assertTrue('INFO [-]' in stdout) + self.assertTrue('DEBUG [-]' in stdout) + self.assertTrue('AUDIT [-]' in stdout) + + # 3. AUDIT log level - audit messages should be included + process = self._start_process(config_path=ST2_CONFIG_AUDIT_LL_PATH) + self.add_process(process=process) + + # Give it some time to start up + eventlet.sleep(3) + process.send_signal(signal.SIGKILL) + + # First 3 log lines are debug messages about the environment which are always logged + stdout = '\n'.join(process.stdout.read().split('\n')[3:]) + + self.assertTrue('INFO [-]' not in stdout) + self.assertTrue('DEBUG [-]' not in stdout) + self.assertTrue('AUDIT [-]' in stdout) + + # 2. INFO log level but system.debug set to True + process = self._start_process(config_path=ST2_CONFIG_SYSTEM_DEBUG_PATH) + self.add_process(process=process) + + # Give it some time to start up + eventlet.sleep(3) + process.send_signal(signal.SIGKILL) + + # First 3 log lines are debug messages about the environment which are always logged + stdout = '\n'.join(process.stdout.read().split('\n')[3:]) + + self.assertTrue('INFO [-]' in stdout) + self.assertTrue('DEBUG [-]' in stdout) + self.assertTrue('AUDIT [-]' in stdout) + + def _start_process(self, config_path): + cmd = CMD + [config_path] + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=False, preexec_fn=os.setsid) + return process diff --git a/st2tests/st2tests/fixtures/conf/logging.api.audit.conf b/st2tests/st2tests/fixtures/conf/logging.api.audit.conf new file mode 100644 index 0000000000..3b5f3005f8 --- /dev/null +++ b/st2tests/st2tests/fixtures/conf/logging.api.audit.conf @@ -0,0 +1,44 @@ +[loggers] +keys=root + +[handlers] +keys=consoleHandler, fileHandler, auditHandler + +[formatters] +keys=simpleConsoleFormatter, verboseConsoleFormatter, gelfFormatter + +[logger_root] +level=AUDIT +handlers=consoleHandler, fileHandler, auditHandler + +[handler_consoleHandler] +class=StreamHandler +level=AUDIT +formatter=simpleConsoleFormatter +args=(sys.stdout,) + +[handler_fileHandler] +class=st2common.log.FormatNamedFileHandler +level=AUDIT +formatter=verboseConsoleFormatter +args=("/tmp/st2api.{timestamp}.log",) + +[handler_auditHandler] +class=st2common.log.FormatNamedFileHandler +level=AUDIT +formatter=gelfFormatter +args=("/tmp/st2api.audit.{timestamp}.log",) + +[formatter_simpleConsoleFormatter] +class=st2common.logging.formatters.ConsoleLogFormatter +format=%(asctime)s %(levelname)s [-] %(message)s +datefmt= + +[formatter_verboseConsoleFormatter] +class=st2common.logging.formatters.ConsoleLogFormatter +format=%(asctime)s %(thread)s %(levelname)s %(module)s [-] %(message)s +datefmt= + +[formatter_gelfFormatter] +class=st2common.logging.formatters.GelfLogFormatter +format=%(message)s diff --git a/st2tests/st2tests/fixtures/conf/logging.api.debug.conf b/st2tests/st2tests/fixtures/conf/logging.api.debug.conf new file mode 100644 index 0000000000..1d7e8ca7ed --- /dev/null +++ b/st2tests/st2tests/fixtures/conf/logging.api.debug.conf @@ -0,0 +1,44 @@ +[loggers] +keys=root + +[handlers] +keys=consoleHandler, fileHandler, auditHandler + +[formatters] +keys=simpleConsoleFormatter, verboseConsoleFormatter, gelfFormatter + +[logger_root] +level=DEBUG +handlers=consoleHandler, fileHandler, auditHandler + +[handler_consoleHandler] +class=StreamHandler +level=DEBUG +formatter=simpleConsoleFormatter +args=(sys.stdout,) + +[handler_fileHandler] +class=st2common.log.FormatNamedFileHandler +level=DEBUG +formatter=verboseConsoleFormatter +args=("/tmp/st2api.{timestamp}.log",) + +[handler_auditHandler] +class=st2common.log.FormatNamedFileHandler +level=AUDIT +formatter=gelfFormatter +args=("/tmp/st2api.audit.{timestamp}.log",) + +[formatter_simpleConsoleFormatter] +class=st2common.logging.formatters.ConsoleLogFormatter +format=%(asctime)s %(levelname)s [-] %(message)s +datefmt= + +[formatter_verboseConsoleFormatter] +class=st2common.logging.formatters.ConsoleLogFormatter +format=%(asctime)s %(thread)s %(levelname)s %(module)s [-] %(message)s +datefmt= + +[formatter_gelfFormatter] +class=st2common.logging.formatters.GelfLogFormatter +format=%(message)s diff --git a/st2tests/st2tests/fixtures/conf/logging.api.info.conf b/st2tests/st2tests/fixtures/conf/logging.api.info.conf new file mode 100644 index 0000000000..f035bcdcb6 --- /dev/null +++ b/st2tests/st2tests/fixtures/conf/logging.api.info.conf @@ -0,0 +1,44 @@ +[loggers] +keys=root + +[handlers] +keys=consoleHandler, fileHandler, auditHandler + +[formatters] +keys=simpleConsoleFormatter, verboseConsoleFormatter, gelfFormatter + +[logger_root] +level=INFO +handlers=consoleHandler, fileHandler, auditHandler + +[handler_consoleHandler] +class=StreamHandler +level=INFO +formatter=simpleConsoleFormatter +args=(sys.stdout,) + +[handler_fileHandler] +class=st2common.log.FormatNamedFileHandler +level=INFO +formatter=verboseConsoleFormatter +args=("/tmp/st2api.{timestamp}.log",) + +[handler_auditHandler] +class=st2common.log.FormatNamedFileHandler +level=AUDIT +formatter=gelfFormatter +args=("/tmp/st2api.audit.{timestamp}.log",) + +[formatter_simpleConsoleFormatter] +class=st2common.logging.formatters.ConsoleLogFormatter +format=%(asctime)s %(levelname)s [-] %(message)s +datefmt= + +[formatter_verboseConsoleFormatter] +class=st2common.logging.formatters.ConsoleLogFormatter +format=%(asctime)s %(thread)s %(levelname)s %(module)s [-] %(message)s +datefmt= + +[formatter_gelfFormatter] +class=st2common.logging.formatters.GelfLogFormatter +format=%(message)s diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.audit_log_level.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.audit_log_level.conf new file mode 100644 index 0000000000..dbe36c36db --- /dev/null +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.audit_log_level.conf @@ -0,0 +1,99 @@ +# Config file used by integration tests + +[database] +db_name = st2-test + +[api] +# Host and port to bind the API server. +host = 127.0.0.1 +port = 9101 +logging = st2tests/st2tests/fixtures/conf/logging.api.audit.conf +mask_secrets = False +# allow_origin is required for handling CORS in st2 web UI. +# allow_origin = http://myhost1.example.com:3000,http://myhost2.example.com:3000 + +[sensorcontainer] +logging = st2tests/conf/logging.sensorcontainer.conf +sensor_node_name = sensornode1 +partition_provider = name:default + +[rulesengine] +logging = st2reactor/conf/logging.rulesengine.conf + +[timersengine] +logging = st2reactor/conf/logging.timersengine.conf + +[actionrunner] +logging = st2actions/conf/logging.conf + +[auth] +host = 127.0.0.1 +port = 9100 +use_ssl = False +debug = False +enable = False +logging = st2tests/conf/logging.auth.conf + +mode = standalone +backend = flat_file +backend_kwargs = {"file_path": "st2auth/conf/htpasswd_dev"} + +# Base URL to the API endpoint excluding the version (e.g. http://myhost.net:9101/) +api_url = http://127.0.0.1:9101/ + +[system] +debug = False +# This way integration tests can write to this directory +base_path = /tmp + +[garbagecollector] +logging = st2reactor/conf/logging.garbagecollector.conf + +action_executions_ttl = 20 +action_executions_output_ttl = 10 +trigger_instances_ttl = 20 +purge_inquiries = True + +collection_interval = 1 +sleep_delay = 0.1 + +[content] +system_packs_base_path = +packs_base_paths = st2tests/st2tests/fixtures/packs/ + +[syslog] +host = 127.0.0.1 +port = 514 +facility = local7 +protocol = udp + +[webui] +# webui_base_url = https://mywebhost.domain + +[log] +excludes = requests,paramiko +redirect_stderr = False +mask_secrets = False + +[system_user] +user = stanley +ssh_key_file = /home/vagrant/.ssh/stanley_rsa + +[messaging] +url = amqp://guest:guest@127.0.0.1:5672/ + +[ssh_runner] +remote_dir = /tmp + +[resultstracker] +logging = st2actions/conf/logging.resultstracker.conf +query_interval = 0.1 + +[notifier] +logging = st2actions/conf/logging.notifier.conf + +[exporter] +logging = st2exporter/conf/logging.exporter.conf + +[mistral] +jitter_interval = 0 diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.debug_log_level.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.debug_log_level.conf new file mode 100644 index 0000000000..caad395240 --- /dev/null +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.debug_log_level.conf @@ -0,0 +1,99 @@ +# Config file used by integration tests + +[database] +db_name = st2-test + +[api] +# Host and port to bind the API server. +host = 127.0.0.1 +port = 9101 +logging = st2tests/st2tests/fixtures/conf/logging.api.debug.conf +mask_secrets = False +# allow_origin is required for handling CORS in st2 web UI. +# allow_origin = http://myhost1.example.com:3000,http://myhost2.example.com:3000 + +[sensorcontainer] +logging = st2tests/conf/logging.sensorcontainer.conf +sensor_node_name = sensornode1 +partition_provider = name:default + +[rulesengine] +logging = st2reactor/conf/logging.rulesengine.conf + +[timersengine] +logging = st2reactor/conf/logging.timersengine.conf + +[actionrunner] +logging = st2actions/conf/logging.conf + +[auth] +host = 127.0.0.1 +port = 9100 +use_ssl = False +debug = False +enable = False +logging = st2tests/conf/logging.auth.conf + +mode = standalone +backend = flat_file +backend_kwargs = {"file_path": "st2auth/conf/htpasswd_dev"} + +# Base URL to the API endpoint excluding the version (e.g. http://myhost.net:9101/) +api_url = http://127.0.0.1:9101/ + +[system] +debug = False +# This way integration tests can write to this directory +base_path = /tmp + +[garbagecollector] +logging = st2reactor/conf/logging.garbagecollector.conf + +action_executions_ttl = 20 +action_executions_output_ttl = 10 +trigger_instances_ttl = 20 +purge_inquiries = True + +collection_interval = 1 +sleep_delay = 0.1 + +[content] +system_packs_base_path = +packs_base_paths = st2tests/st2tests/fixtures/packs/ + +[syslog] +host = 127.0.0.1 +port = 514 +facility = local7 +protocol = udp + +[webui] +# webui_base_url = https://mywebhost.domain + +[log] +excludes = requests,paramiko +redirect_stderr = False +mask_secrets = False + +[system_user] +user = stanley +ssh_key_file = /home/vagrant/.ssh/stanley_rsa + +[messaging] +url = amqp://guest:guest@127.0.0.1:5672/ + +[ssh_runner] +remote_dir = /tmp + +[resultstracker] +logging = st2actions/conf/logging.resultstracker.conf +query_interval = 0.1 + +[notifier] +logging = st2actions/conf/logging.notifier.conf + +[exporter] +logging = st2exporter/conf/logging.exporter.conf + +[mistral] +jitter_interval = 0 diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.info_log_level.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.info_log_level.conf new file mode 100644 index 0000000000..5cd4e6cd33 --- /dev/null +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.info_log_level.conf @@ -0,0 +1,99 @@ +# Config file used by integration tests + +[database] +db_name = st2-test + +[api] +# Host and port to bind the API server. +host = 127.0.0.1 +port = 9101 +logging = st2tests/st2tests/fixtures/conf/logging.api.info.conf +mask_secrets = False +# allow_origin is required for handling CORS in st2 web UI. +# allow_origin = http://myhost1.example.com:3000,http://myhost2.example.com:3000 + +[sensorcontainer] +logging = st2tests/conf/logging.sensorcontainer.conf +sensor_node_name = sensornode1 +partition_provider = name:default + +[rulesengine] +logging = st2reactor/conf/logging.rulesengine.conf + +[timersengine] +logging = st2reactor/conf/logging.timersengine.conf + +[actionrunner] +logging = st2actions/conf/logging.conf + +[auth] +host = 127.0.0.1 +port = 9100 +use_ssl = False +debug = False +enable = False +logging = st2tests/conf/logging.auth.conf + +mode = standalone +backend = flat_file +backend_kwargs = {"file_path": "st2auth/conf/htpasswd_dev"} + +# Base URL to the API endpoint excluding the version (e.g. http://myhost.net:9101/) +api_url = http://127.0.0.1:9101/ + +[system] +debug = False +# This way integration tests can write to this directory +base_path = /tmp + +[garbagecollector] +logging = st2reactor/conf/logging.garbagecollector.conf + +action_executions_ttl = 20 +action_executions_output_ttl = 10 +trigger_instances_ttl = 20 +purge_inquiries = True + +collection_interval = 1 +sleep_delay = 0.1 + +[content] +system_packs_base_path = +packs_base_paths = st2tests/st2tests/fixtures/packs/ + +[syslog] +host = 127.0.0.1 +port = 514 +facility = local7 +protocol = udp + +[webui] +# webui_base_url = https://mywebhost.domain + +[log] +excludes = requests,paramiko +redirect_stderr = False +mask_secrets = False + +[system_user] +user = stanley +ssh_key_file = /home/vagrant/.ssh/stanley_rsa + +[messaging] +url = amqp://guest:guest@127.0.0.1:5672/ + +[ssh_runner] +remote_dir = /tmp + +[resultstracker] +logging = st2actions/conf/logging.resultstracker.conf +query_interval = 0.1 + +[notifier] +logging = st2actions/conf/logging.notifier.conf + +[exporter] +logging = st2exporter/conf/logging.exporter.conf + +[mistral] +jitter_interval = 0 diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true.conf b/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true.conf new file mode 100644 index 0000000000..3317c11abf --- /dev/null +++ b/st2tests/st2tests/fixtures/conf/st2.tests.api.system_debug_true.conf @@ -0,0 +1,99 @@ +# Config file used by integration tests + +[database] +db_name = st2-test + +[api] +# Host and port to bind the API server. +host = 127.0.0.1 +port = 9101 +logging = st2tests/st2tests/fixtures/conf/logging.api.info.conf +mask_secrets = False +# allow_origin is required for handling CORS in st2 web UI. +# allow_origin = http://myhost1.example.com:3000,http://myhost2.example.com:3000 + +[sensorcontainer] +logging = st2tests/conf/logging.sensorcontainer.conf +sensor_node_name = sensornode1 +partition_provider = name:default + +[rulesengine] +logging = st2reactor/conf/logging.rulesengine.conf + +[timersengine] +logging = st2reactor/conf/logging.timersengine.conf + +[actionrunner] +logging = st2actions/conf/logging.conf + +[auth] +host = 127.0.0.1 +port = 9100 +use_ssl = False +debug = False +enable = False +logging = st2tests/conf/logging.auth.conf + +mode = standalone +backend = flat_file +backend_kwargs = {"file_path": "st2auth/conf/htpasswd_dev"} + +# Base URL to the API endpoint excluding the version (e.g. http://myhost.net:9101/) +api_url = http://127.0.0.1:9101/ + +[system] +debug = True +# This way integration tests can write to this directory +base_path = /tmp + +[garbagecollector] +logging = st2reactor/conf/logging.garbagecollector.conf + +action_executions_ttl = 20 +action_executions_output_ttl = 10 +trigger_instances_ttl = 20 +purge_inquiries = True + +collection_interval = 1 +sleep_delay = 0.1 + +[content] +system_packs_base_path = +packs_base_paths = st2tests/st2tests/fixtures/packs/ + +[syslog] +host = 127.0.0.1 +port = 514 +facility = local7 +protocol = udp + +[webui] +# webui_base_url = https://mywebhost.domain + +[log] +excludes = requests,paramiko +redirect_stderr = False +mask_secrets = False + +[system_user] +user = stanley +ssh_key_file = /home/vagrant/.ssh/stanley_rsa + +[messaging] +url = amqp://guest:guest@127.0.0.1:5672/ + +[ssh_runner] +remote_dir = /tmp + +[resultstracker] +logging = st2actions/conf/logging.resultstracker.conf +query_interval = 0.1 + +[notifier] +logging = st2actions/conf/logging.notifier.conf + +[exporter] +logging = st2exporter/conf/logging.exporter.conf + +[mistral] +jitter_interval = 0 diff --git a/st2tests/st2tests/fixtures/conf/st2.tests.conf b/st2tests/st2tests/fixtures/conf/st2.tests.conf new file mode 100644 index 0000000000..bb97039f03 --- /dev/null +++ b/st2tests/st2tests/fixtures/conf/st2.tests.conf @@ -0,0 +1,100 @@ +# Config file used by integration tests + +[database] +db_name = st2-test + +[api] +# Host and port to bind the API server. +host = 127.0.0.1 +port = 9101 +logging = st2tests/conf/logging.api.conf +mask_secrets = False +# allow_origin is required for handling CORS in st2 web UI. +# allow_origin = http://myhost1.example.com:3000,http://myhost2.example.com:3000 + +[sensorcontainer] +logging = st2tests/conf/logging.sensorcontainer.conf +sensor_node_name = sensornode1 +partition_provider = name:default + +[rulesengine] +logging = st2reactor/conf/logging.rulesengine.conf + +[timersengine] +logging = st2reactor/conf/logging.timersengine.conf + +[actionrunner] +logging = st2actions/conf/logging.conf + +[auth] +host = 127.0.0.1 +port = 9100 +use_ssl = False +debug = False +enable = False +logging = st2tests/conf/logging.auth.conf + +mode = standalone +backend = flat_file +backend_kwargs = {"file_path": "st2auth/conf/htpasswd_dev"} + +# Base URL to the API endpoint excluding the version (e.g. http://myhost.net:9101/) +api_url = http://127.0.0.1:9101/ + +[system] +debug = False + +# This way integration tests can write to this directory +base_path = /tmp + +[garbagecollector] +logging = st2reactor/conf/logging.garbagecollector.conf + +action_executions_ttl = 20 +action_executions_output_ttl = 10 +trigger_instances_ttl = 20 +purge_inquiries = True + +collection_interval = 1 +sleep_delay = 0.1 + +[content] +system_packs_base_path = +packs_base_paths = st2tests/st2tests/fixtures/packs/ + +[syslog] +host = 127.0.0.1 +port = 514 +facility = local7 +protocol = udp + +[webui] +# webui_base_url = https://mywebhost.domain + +[log] +excludes = requests,paramiko +redirect_stderr = False +mask_secrets = False + +[system_user] +user = stanley +ssh_key_file = /home/vagrant/.ssh/stanley_rsa + +[messaging] +url = amqp://guest:guest@127.0.0.1:5672/ + +[ssh_runner] +remote_dir = /tmp + +[resultstracker] +logging = st2actions/conf/logging.resultstracker.conf +query_interval = 0.1 + +[notifier] +logging = st2actions/conf/logging.notifier.conf + +[exporter] +logging = st2exporter/conf/logging.exporter.conf + +[mistral] +jitter_interval = 0 From 06243c7048d49281c3e0d84685a1659e088de0a9 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 6 Feb 2019 13:48:54 +0100 Subject: [PATCH 5/6] Use longer sleep to avoid false negatives. --- .../integration/test_service_setup_log_level_filtering.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/st2common/tests/integration/test_service_setup_log_level_filtering.py b/st2common/tests/integration/test_service_setup_log_level_filtering.py index 077d4a9f38..1e177dec45 100644 --- a/st2common/tests/integration/test_service_setup_log_level_filtering.py +++ b/st2common/tests/integration/test_service_setup_log_level_filtering.py @@ -76,7 +76,7 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): self.add_process(process=process) # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(5) process.send_signal(signal.SIGKILL) # First 3 log lines are debug messages about the environment which are always logged @@ -91,7 +91,7 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): self.add_process(process=process) # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(5) process.send_signal(signal.SIGKILL) # First 3 log lines are debug messages about the environment which are always logged @@ -106,7 +106,7 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): self.add_process(process=process) # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(5) process.send_signal(signal.SIGKILL) # First 3 log lines are debug messages about the environment which are always logged From 4e4371f907c0287a3f688fa07b1cf57b5662edf1 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 6 Feb 2019 13:51:55 +0100 Subject: [PATCH 6/6] Fix Python 3 compatibility. --- .../integration/test_service_setup_log_level_filtering.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/st2common/tests/integration/test_service_setup_log_level_filtering.py b/st2common/tests/integration/test_service_setup_log_level_filtering.py index 1e177dec45..432834d486 100644 --- a/st2common/tests/integration/test_service_setup_log_level_filtering.py +++ b/st2common/tests/integration/test_service_setup_log_level_filtering.py @@ -65,7 +65,7 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): process.send_signal(signal.SIGKILL) # First 3 log lines are debug messages about the environment which are always logged - stdout = '\n'.join(process.stdout.read().split('\n')[3:]) + stdout = '\n'.join(process.stdout.read().decode('utf-8').split('\n')[3:]) self.assertTrue('INFO [-]' in stdout) self.assertTrue('DEBUG [-]' not in stdout) @@ -80,7 +80,7 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): process.send_signal(signal.SIGKILL) # First 3 log lines are debug messages about the environment which are always logged - stdout = '\n'.join(process.stdout.read().split('\n')[3:]) + stdout = '\n'.join(process.stdout.read().decode('utf-8').split('\n')[3:]) self.assertTrue('INFO [-]' in stdout) self.assertTrue('DEBUG [-]' in stdout) @@ -95,7 +95,7 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): process.send_signal(signal.SIGKILL) # First 3 log lines are debug messages about the environment which are always logged - stdout = '\n'.join(process.stdout.read().split('\n')[3:]) + stdout = '\n'.join(process.stdout.read().decode('utf-8').split('\n')[3:]) self.assertTrue('INFO [-]' not in stdout) self.assertTrue('DEBUG [-]' not in stdout) @@ -110,7 +110,7 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): process.send_signal(signal.SIGKILL) # First 3 log lines are debug messages about the environment which are always logged - stdout = '\n'.join(process.stdout.read().split('\n')[3:]) + stdout = '\n'.join(process.stdout.read().decode('utf-8').split('\n')[3:]) self.assertTrue('INFO [-]' in stdout) self.assertTrue('DEBUG [-]' in stdout)