diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e3fa1c7904..ac8c8beb8e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -55,6 +55,22 @@ Changed Contributed by @nzlosh +* Add new ``--enable-profiler`` flag to all the servies. This flag enables cProfiler based profiler + for the service in question and dumps the profiling data to a file on process + exit. + + This functionality should never be used in production, but only in development environments or + similar when profiling code. #5199 + + Contributed by @Kami. + +* Add new ``--enable-eventlet-blocking-detection`` flag to all the servies. This flag enables + eventlet long operation / blocked main loop logic which throws an exception if a particular + code blocks longer than a specific duration in seconds. + + This functionality should never be used in production, but only in development environments or + similar when debugging code. #5199 + Fixed ~~~~~ diff --git a/st2client/st2client/shell.py b/st2client/st2client/shell.py index 4bca31e753..5c3a6b9cd8 100755 --- a/st2client/st2client/shell.py +++ b/st2client/st2client/shell.py @@ -66,7 +66,7 @@ from st2client.utils.misc import reencode_list_with_surrogate_escape_sequences from st2client.commands.auth import TokenCreateCommand from st2client.commands.auth import LoginCommand - +from st2client.utils.profiler import setup_regular_profiler __all__ = ["Shell"] @@ -532,6 +532,12 @@ def setup_logging(argv): def main(argv=sys.argv[1:]): setup_logging(argv) + + if "--enable-profiler" in sys.argv: + setup_regular_profiler(service_name="st2cli") + sys.argv.remove("--enable-profiler") + argv.remove("--enable-profiler") + return Shell().run(argv) diff --git a/st2client/st2client/utils/profiler.py b/st2client/st2client/utils/profiler.py new file mode 100644 index 0000000000..524e441ac9 --- /dev/null +++ b/st2client/st2client/utils/profiler.py @@ -0,0 +1,46 @@ +# Copyright 2020 The StackStorm Authors. +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import atexit +import platform +import cProfile + +__all__ = ["setup_regular_profiler"] + + +def setup_regular_profiler(service_name: str) -> None: + """ + Set up regular Python cProf profiler and write result to a file on exit. + """ + profiler = cProfile.Profile() + profiler.enable() + + file_path = os.path.join( + "/tmp", "%s-%s-%s.cprof" % (service_name, platform.machine(), int(time.time())) + ) + + print("Eventlet profiler enabled") + print("Profiling data will be saved to %s on exit" % (file_path)) + + def stop_profiler(): + profiler.disable() + profiler.dump_stats(file_path) + print("Profiling data written to %s" % (file_path)) + print("You can view it using: ") + print("\t python3 -m pstats %s" % (file_path)) + + atexit.register(stop_profiler) diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index cf2ada4ee3..0c002a8e2f 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -632,6 +632,21 @@ def register_opts(ignore_errors=False): "eventlet library is used to support async IO. This could result in " "failures that do not occur under normal operation.", ), + cfg.BoolOpt( + "enable-profiler", + default=False, + help="Enable code profiler mode. Do not use in production.", + ), + cfg.BoolOpt( + "enable-eventlet-blocking-detection", + default=False, + help="Enable eventlet blocking detection logic. Do not use in production.", + ), + cfg.FloatOpt( + "eventlet-blocking-detection-resolution", + default=0.5, + help="Resolution in seconds for eventlet blocking detection logic.", + ), ] do_register_cli_opts(cli_opts, ignore_errors=ignore_errors) diff --git a/st2common/st2common/service_setup.py b/st2common/st2common/service_setup.py index 44bfc3d4da..2c87c8fcaa 100644 --- a/st2common/st2common/service_setup.py +++ b/st2common/st2common/service_setup.py @@ -26,6 +26,7 @@ import logging as stdlib_logging import six +import eventlet.debug from oslo_config import cfg from tooz.coordination import GroupAlreadyExist @@ -45,6 +46,7 @@ from st2common.logging.misc import add_global_filters_for_all_loggers from st2common.constants.error_messages import PYTHON2_DEPRECATION from st2common.services.coordination import get_driver_name +from st2common.util.profiler import setup_eventlet_profiler # Note: This is here for backward compatibility. # Function has been moved in a standalone module to avoid expensive in-direct @@ -121,6 +123,9 @@ def setup( else: config.parse_args() + if cfg.CONF.enable_profiler: + setup_eventlet_profiler(service_name="st2" + service) + version = "%s.%s.%s" % ( sys.version_info[0], sys.version_info[1], @@ -273,6 +278,17 @@ def setup( if sys.version_info[0] == 2: LOG.warning(PYTHON2_DEPRECATION) + # NOTE: This must be called here at the end of the setup phase since some of the setup code and + # modules like jinja, stevedore, etc load files from disk on init which is slow and will be + # detected as blocking operation, but this is not really an issue inside the service startup / + # init phase. + if cfg.CONF.enable_eventlet_blocking_detection: + print("Eventlet long running / blocking operation detection logic enabled") + print(cfg.CONF.eventlet_blocking_detection_resolution) + eventlet.debug.hub_blocking_detection( + state=True, resolution=cfg.CONF.eventlet_blocking_detection_resolution + ) + def teardown(): """ diff --git a/st2common/st2common/util/profiler.py b/st2common/st2common/util/profiler.py new file mode 100644 index 0000000000..b8760a2046 --- /dev/null +++ b/st2common/st2common/util/profiler.py @@ -0,0 +1,81 @@ +# Copyright 2020 The StackStorm Authors. +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import atexit +import platform +import cProfile + +import eventlet +from eventlet.green import profile + +__all__ = ["setup_regular_profiler", "setup_eventlet_profiler"] + + +def setup_regular_profiler(service_name: str) -> None: + """ + Set up regular Python cProf profiler and write result to a file on exit. + """ + profiler = cProfile.Profile() + profiler.enable() + + file_path = os.path.join( + "/tmp", "%s-%s-%s.cprof" % (service_name, platform.machine(), int(time.time())) + ) + + print("Eventlet profiler enabled") + print("Profiling data will be saved to %s on exit" % (file_path)) + + def stop_profiler(): + profiler.disable() + profiler.dump_stats(file_path) + print("Profiling data written to %s" % (file_path)) + print("You can view it using: ") + print("\t python3 -m pstats %s" % (file_path)) + + atexit.register(stop_profiler) + + +def setup_eventlet_profiler(service_name: str) -> None: + """ + Set up eventlet profiler and write results to a file on exit. + + Only to be used with eventlet code (aka an StackStorm service minus the CLI). + """ + is_patched = eventlet.patcher.is_monkey_patched("os") + if not is_patched: + raise ValueError( + "No eventlet monkey patching detected. Code may not be using eventlet" + ) + + profiler = profile.Profile() + profiler.start() + + file_path = os.path.join( + "/tmp", "%s-%s-%s.cprof" % (service_name, platform.machine(), int(time.time())) + ) + + print("Eventlet profiler enabled") + print("Profiling data will be saved to %s on exit" % (file_path)) + + def stop_profiler(): + profiler.stop() + profiler.dump_stats(file_path) + print("Profiling data written to %s" % (file_path)) + print("You can view it using: ") + print("\t python3 -m pstats %s" % (file_path)) + + atexit.register(stop_profiler)