Add "--enable-profiler" flag to all the StackStorm services + CLI

CHANGELOG.rst

@@ -55,6 +55,22 @@ Changed
 
   Contributed by @nzlosh
 
+* Add new ``--enable-profiler`` flag to all the servies. This flag enables cProfiler based profiler
+  for the service in question and  dumps the profiling data to a file on process
+  exit.
+
+  This functionality should never be used in production, but only in development environments or
+  similar when profiling code. #5199
+
+  Contributed by @Kami.
+
+* Add new ``--enable-eventlet-blocking-detection`` flag to all the servies. This flag enables
+  eventlet long operation / blocked main loop logic which throws an exception if a particular
+  code blocks longer than a specific duration in seconds.
+
+  This functionality should never be used in production, but only in development environments or
+  similar when debugging code. #5199
+
 Fixed
 ~~~~~
 

st2client/st2client/shell.py

@@ -66,7 +66,7 @@
 from st2client.utils.misc import reencode_list_with_surrogate_escape_sequences
 from st2client.commands.auth import TokenCreateCommand
 from st2client.commands.auth import LoginCommand
-
+from st2client.utils.profiler import setup_regular_profiler
 
 __all__ = ["Shell"]
 
@@ -532,6 +532,12 @@ def setup_logging(argv):
 
 def main(argv=sys.argv[1:]):
     setup_logging(argv)
+
+    if "--enable-profiler" in sys.argv:
+        setup_regular_profiler(service_name="st2cli")
+        sys.argv.remove("--enable-profiler")
+        argv.remove("--enable-profiler")
+
     return Shell().run(argv)
 
 

st2client/st2client/utils/profiler.py

@@ -0,0 +1,46 @@
+# Copyright 2020 The StackStorm Authors.
+# Copyright 2019 Extreme Networks, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+import atexit
+import platform
+import cProfile
+
+__all__ = ["setup_regular_profiler"]
+
+
+def setup_regular_profiler(service_name: str) -> None:
+    """
+    Set up regular Python cProf profiler and write result to a file on exit.
+    """
+    profiler = cProfile.Profile()
+    profiler.enable()
+
+    file_path = os.path.join(
+        "/tmp", "%s-%s-%s.cprof" % (service_name, platform.machine(), int(time.time()))
+    )
+
+    print("Eventlet profiler enabled")
+    print("Profiling data will be saved to %s on exit" % (file_path))
+
+    def stop_profiler():
+        profiler.disable()
+        profiler.dump_stats(file_path)
+        print("Profiling data written to %s" % (file_path))
+        print("You can view it using: ")
+        print("\t python3 -m pstats %s" % (file_path))
+
+    atexit.register(stop_profiler)

st2common/st2common/config.py

@@ -632,6 +632,21 @@ def register_opts(ignore_errors=False):
             "eventlet library is used to support async IO. This could result in "
             "failures that do not occur under normal operation.",
         ),
+        cfg.BoolOpt(
+            "enable-profiler",
+            default=False,
+            help="Enable code profiler mode. Do not use in production.",
+        ),
+        cfg.BoolOpt(
+            "enable-eventlet-blocking-detection",
+            default=False,
+            help="Enable eventlet blocking detection logic. Do not use in production.",
+        ),
+        cfg.FloatOpt(
+            "eventlet-blocking-detection-resolution",
+            default=0.5,
+            help="Resolution in seconds for eventlet blocking detection logic.",
+        ),
     ]
 
     do_register_cli_opts(cli_opts, ignore_errors=ignore_errors)

st2common/st2common/service_setup.py

@@ -26,6 +26,7 @@
 import logging as stdlib_logging
 
 import six
+import eventlet.debug
 from oslo_config import cfg
 from tooz.coordination import GroupAlreadyExist
 
@@ -45,6 +46,7 @@
 from st2common.logging.misc import add_global_filters_for_all_loggers
 from st2common.constants.error_messages import PYTHON2_DEPRECATION
 from st2common.services.coordination import get_driver_name
+from st2common.util.profiler import setup_eventlet_profiler
 
 # Note: This is here for backward compatibility.
 # Function has been moved in a standalone module to avoid expensive in-direct
@@ -121,6 +123,9 @@ def setup(
     else:
         config.parse_args()
 
+    if cfg.CONF.enable_profiler:
+        setup_eventlet_profiler(service_name="st2" + service)
+
     version = "%s.%s.%s" % (
         sys.version_info[0],
         sys.version_info[1],
@@ -273,6 +278,17 @@ def setup(
     if sys.version_info[0] == 2:
         LOG.warning(PYTHON2_DEPRECATION)
 
+    # NOTE: This must be called here at the end of the setup phase since some of the setup code and
+    # modules like jinja, stevedore, etc load files from disk on init which is slow and will be
+    # detected as blocking operation, but this is not really an issue inside the service startup /
+    # init phase.
+    if cfg.CONF.enable_eventlet_blocking_detection:
+        print("Eventlet long running / blocking operation detection logic enabled")
+        print(cfg.CONF.eventlet_blocking_detection_resolution)
+        eventlet.debug.hub_blocking_detection(
+            state=True, resolution=cfg.CONF.eventlet_blocking_detection_resolution
+        )
+
 
 def teardown():
     """

st2common/st2common/util/profiler.py

@@ -0,0 +1,81 @@
+# Copyright 2020 The StackStorm Authors.
+# Copyright 2019 Extreme Networks, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+import atexit
+import platform
+import cProfile
+
+import eventlet
+from eventlet.green import profile
+
+__all__ = ["setup_regular_profiler", "setup_eventlet_profiler"]
+
+
+def setup_regular_profiler(service_name: str) -> None:
+    """
+    Set up regular Python cProf profiler and write result to a file on exit.
+    """
+    profiler = cProfile.Profile()
+    profiler.enable()
+
+    file_path = os.path.join(
+        "/tmp", "%s-%s-%s.cprof" % (service_name, platform.machine(), int(time.time()))
+    )
+
+    print("Eventlet profiler enabled")
+    print("Profiling data will be saved to %s on exit" % (file_path))
+
+    def stop_profiler():
+        profiler.disable()
+        profiler.dump_stats(file_path)
+        print("Profiling data written to %s" % (file_path))
+        print("You can view it using: ")
+        print("\t python3 -m pstats %s" % (file_path))
+
+    atexit.register(stop_profiler)
+
+
+def setup_eventlet_profiler(service_name: str) -> None:
+    """
+    Set up eventlet profiler and write results to a file on exit.
+
+    Only to be used with eventlet code (aka an StackStorm service minus the CLI).
+    """
+    is_patched = eventlet.patcher.is_monkey_patched("os")
+    if not is_patched:
+        raise ValueError(
+            "No eventlet monkey patching detected. Code may not be using eventlet"
+        )
+
+    profiler = profile.Profile()
+    profiler.start()
+
+    file_path = os.path.join(
+        "/tmp", "%s-%s-%s.cprof" % (service_name, platform.machine(), int(time.time()))
+    )
+
+    print("Eventlet profiler enabled")
+    print("Profiling data will be saved to %s on exit" % (file_path))
+
+    def stop_profiler():
+        profiler.stop()
+        profiler.dump_stats(file_path)
+        print("Profiling data written to %s" % (file_path))
+        print("You can view it using: ")
+        print("\t python3 -m pstats %s" % (file_path))
+
+    atexit.register(stop_profiler)