From aa0d3b6dbf5340f3653932724599d94a963199af Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 21 Mar 2021 19:12:53 +0100 Subject: [PATCH 1/5] Add --enable-profiler CLI flag to all the StackStorm services + CLI. This flag can be used to enable cProf based profiler for debugging and troubleshooting purposes. --- st2client/st2client/shell.py | 8 ++- st2client/st2client/utils/profiler.py | 46 +++++++++++++++ st2common/st2common/service_setup.py | 5 ++ st2common/st2common/util/profiler.py | 81 +++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 st2client/st2client/utils/profiler.py create mode 100644 st2common/st2common/util/profiler.py diff --git a/st2client/st2client/shell.py b/st2client/st2client/shell.py index 7d3359c532..c9f61eb6f7 100755 --- a/st2client/st2client/shell.py +++ b/st2client/st2client/shell.py @@ -65,7 +65,7 @@ from st2client.utils.logging import LogLevelFilter, set_log_level_for_all_loggers from st2client.commands.auth import TokenCreateCommand from st2client.commands.auth import LoginCommand - +from st2client.utils.profiler import setup_regular_profiler __all__ = ["Shell"] @@ -495,6 +495,12 @@ def setup_logging(argv): def main(argv=sys.argv[1:]): setup_logging(argv) + + if "--enable-profiler" in sys.argv: + setup_regular_profiler(service_name="st2cli") + sys.argv.remove("--enable-profiler") + argv.remove("--enable-profiler") + return Shell().run(argv) diff --git a/st2client/st2client/utils/profiler.py b/st2client/st2client/utils/profiler.py new file mode 100644 index 0000000000..524e441ac9 --- /dev/null +++ b/st2client/st2client/utils/profiler.py @@ -0,0 +1,46 @@ +# Copyright 2020 The StackStorm Authors. +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import atexit +import platform +import cProfile + +__all__ = ["setup_regular_profiler"] + + +def setup_regular_profiler(service_name: str) -> None: + """ + Set up regular Python cProf profiler and write result to a file on exit. + """ + profiler = cProfile.Profile() + profiler.enable() + + file_path = os.path.join( + "/tmp", "%s-%s-%s.cprof" % (service_name, platform.machine(), int(time.time())) + ) + + print("Eventlet profiler enabled") + print("Profiling data will be saved to %s on exit" % (file_path)) + + def stop_profiler(): + profiler.disable() + profiler.dump_stats(file_path) + print("Profiling data written to %s" % (file_path)) + print("You can view it using: ") + print("\t python3 -m pstats %s" % (file_path)) + + atexit.register(stop_profiler) diff --git a/st2common/st2common/service_setup.py b/st2common/st2common/service_setup.py index 03f9bcdfa7..6c7bf66524 100644 --- a/st2common/st2common/service_setup.py +++ b/st2common/st2common/service_setup.py @@ -44,6 +44,7 @@ from st2common.services import coordination from st2common.logging.misc import add_global_filters_for_all_loggers from st2common.constants.error_messages import PYTHON2_DEPRECATION +from st2common.util.profiler import setup_eventlet_profiler # Note: This is here for backward compatibility. # Function has been moved in a standalone module to avoid expensive in-direct @@ -95,6 +96,10 @@ def setup( :param service: Name of the service. :param config: Config object to use to parse args. """ + if "--enable-profiler" in sys.argv: + setup_eventlet_profiler(service_name="st2" + service) + sys.argv.remove("--enable-profiler") + capabilities = capabilities or {} # Set up logger which logs everything which happens during and before config diff --git a/st2common/st2common/util/profiler.py b/st2common/st2common/util/profiler.py new file mode 100644 index 0000000000..b8760a2046 --- /dev/null +++ b/st2common/st2common/util/profiler.py @@ -0,0 +1,81 @@ +# Copyright 2020 The StackStorm Authors. +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import atexit +import platform +import cProfile + +import eventlet +from eventlet.green import profile + +__all__ = ["setup_regular_profiler", "setup_eventlet_profiler"] + + +def setup_regular_profiler(service_name: str) -> None: + """ + Set up regular Python cProf profiler and write result to a file on exit. + """ + profiler = cProfile.Profile() + profiler.enable() + + file_path = os.path.join( + "/tmp", "%s-%s-%s.cprof" % (service_name, platform.machine(), int(time.time())) + ) + + print("Eventlet profiler enabled") + print("Profiling data will be saved to %s on exit" % (file_path)) + + def stop_profiler(): + profiler.disable() + profiler.dump_stats(file_path) + print("Profiling data written to %s" % (file_path)) + print("You can view it using: ") + print("\t python3 -m pstats %s" % (file_path)) + + atexit.register(stop_profiler) + + +def setup_eventlet_profiler(service_name: str) -> None: + """ + Set up eventlet profiler and write results to a file on exit. + + Only to be used with eventlet code (aka an StackStorm service minus the CLI). + """ + is_patched = eventlet.patcher.is_monkey_patched("os") + if not is_patched: + raise ValueError( + "No eventlet monkey patching detected. Code may not be using eventlet" + ) + + profiler = profile.Profile() + profiler.start() + + file_path = os.path.join( + "/tmp", "%s-%s-%s.cprof" % (service_name, platform.machine(), int(time.time())) + ) + + print("Eventlet profiler enabled") + print("Profiling data will be saved to %s on exit" % (file_path)) + + def stop_profiler(): + profiler.stop() + profiler.dump_stats(file_path) + print("Profiling data written to %s" % (file_path)) + print("You can view it using: ") + print("\t python3 -m pstats %s" % (file_path)) + + atexit.register(stop_profiler) From a8e7f6410c35dcc15194313cd23830ce82088489 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 17 Apr 2021 11:48:17 +0200 Subject: [PATCH 2/5] Also add flag for enabling eventlet blocking detection logic, utilize oslo config for parsing CLI options. --- CHANGELOG.rst | 18 ++++++++++++++++++ st2common/st2common/config.py | 15 +++++++++++++++ st2common/st2common/service_setup.py | 19 +++++++++++++++---- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9a3c2a11bc..fd6cbc9cc3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -49,6 +49,24 @@ Added Contributed by @cognifloyd. +* Add new ``--enable-profiler`` flag to all the servies. This flag enables cProfiler based profiler + for the service in question and dumps the profiling data to a file on process + exit. + + This functionality should never be used in production, but only in development environments or + similar when profiling code. #5199 + + Contributed by @Kami. + +* Add new ``--enable-eventlet-blocking-detection`` flag to all the servies. This flag enables + eventlet long operation / blocked main loop logic which throws an exception if a particular + code blocks longer than a specific duration in seconds. + + This functionality should never be used in production, but only in development environments or + similar when debugging code. #5199 + + Contributed by @Kami. + Changed ~~~~~~~ diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index 0429c81ea2..d14bc862d8 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -611,6 +611,21 @@ def register_opts(ignore_errors=False): "eventlet library is used to support async IO. This could result in " "failures that do not occur under normal operation.", ), + cfg.BoolOpt( + "enable-profiler", + default=False, + help="Enable code profiler mode. Do not use in production.", + ), + cfg.BoolOpt( + "enable-eventlet-blocking-detection", + default=False, + help="Enable eventlet blocking detection logic. Do not use in production.", + ), + cfg.FloatOpt( + "eventlet-blocking-detection-resolution", + default=0.5, + help="Resolution in seconds for eventlet blocking detection logic.", + ), ] do_register_cli_opts(cli_opts, ignore_errors=ignore_errors) diff --git a/st2common/st2common/service_setup.py b/st2common/st2common/service_setup.py index b489189ad2..937f9a61a5 100644 --- a/st2common/st2common/service_setup.py +++ b/st2common/st2common/service_setup.py @@ -26,6 +26,7 @@ import logging as stdlib_logging import six +import eventlet.debug from oslo_config import cfg from tooz.coordination import GroupAlreadyExist @@ -110,10 +111,6 @@ def setup( :param service: Name of the service. :param config: Config object to use to parse args. """ - if "--enable-profiler" in sys.argv: - setup_eventlet_profiler(service_name="st2" + service) - sys.argv.remove("--enable-profiler") - capabilities = capabilities or {} # Set up logger which logs everything which happens during and before config @@ -126,6 +123,9 @@ def setup( else: config.parse_args() + if cfg.CONF.enable_profiler: + setup_eventlet_profiler(service_name="st2" + service) + version = "%s.%s.%s" % ( sys.version_info[0], sys.version_info[1], @@ -276,6 +276,17 @@ def setup( if sys.version_info[0] == 2: LOG.warning(PYTHON2_DEPRECATION) + # NOTE: This must be called here at the end of the setup phase since some of the setup code and + # modules like jinja, stevedore, etc load files from disk on init which is slow and will be + # detected as blocking operation, but this is not really an issue inside the service startup / + # init phase. + if cfg.CONF.enable_eventlet_blocking_detection: + print("Eventlet long running / blocking operation detection logic enabled") + print(cfg.CONF.eventlet_blocking_detection_resolution) + eventlet.debug.hub_blocking_detection( + state=True, resolution=cfg.CONF.eventlet_blocking_detection_resolution + ) + def teardown(): """ From 762516edb65df1401893d7cffb52da1458acf9a6 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 2 Oct 2021 01:04:03 -0500 Subject: [PATCH 3/5] fix whitespace --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f8497ada35..00d5ad5ab5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -129,7 +129,7 @@ Added and when working with executions with large textual results. #5241 Contributed by @Kami. - + * Add new ``--enable-profiler`` flag to all the servies. This flag enables cProfiler based profiler for the service in question and dumps the profiling data to a file on process exit. From 23617b8fe518cb0f0e4c6c7cb349ae6eafde199e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 2 Oct 2021 01:16:36 -0500 Subject: [PATCH 4/5] fix whitespace again --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 00d5ad5ab5..e717bfcf6d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -130,7 +130,7 @@ Added Contributed by @Kami. - * Add new ``--enable-profiler`` flag to all the servies. This flag enables cProfiler based profiler +* Add new ``--enable-profiler`` flag to all the servies. This flag enables cProfiler based profiler for the service in question and dumps the profiling data to a file on process exit. From b88b3fa74ad405d094656a85ccc9712db511e670 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 2 Oct 2021 01:35:15 -0500 Subject: [PATCH 5/5] move changelog to devel section --- CHANGELOG.rst | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e717bfcf6d..36683aa78b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -45,6 +45,22 @@ Changed Contributed by Amanda McGuinness (@amanda11 Ammeon Solutions) +* Add new ``--enable-profiler`` flag to all the servies. This flag enables cProfiler based profiler + for the service in question and dumps the profiling data to a file on process + exit. + + This functionality should never be used in production, but only in development environments or + similar when profiling code. #5199 + + Contributed by @Kami. + +* Add new ``--enable-eventlet-blocking-detection`` flag to all the servies. This flag enables + eventlet long operation / blocked main loop logic which throws an exception if a particular + code blocks longer than a specific duration in seconds. + + This functionality should never be used in production, but only in development environments or + similar when debugging code. #5199 + Fixed ~~~~~ @@ -130,22 +146,6 @@ Added Contributed by @Kami. -* Add new ``--enable-profiler`` flag to all the servies. This flag enables cProfiler based profiler - for the service in question and dumps the profiling data to a file on process - exit. - - This functionality should never be used in production, but only in development environments or - similar when profiling code. #5199 - - Contributed by @Kami. - -* Add new ``--enable-eventlet-blocking-detection`` flag to all the servies. This flag enables - eventlet long operation / blocked main loop logic which throws an exception if a particular - code blocks longer than a specific duration in seconds. - - This functionality should never be used in production, but only in development environments or - similar when debugging code. #5199 - * Mask secrets in output of an action execution in the API if the action has an output schema defined and one or more output parameters are marked as secret. #5250