Don't call eventlet directly inside the st2reactor code

CHANGELOG.rst

@@ -12,8 +12,10 @@ Added
   #4757
 * Add ``user`` parameter to ``re_run`` method of st2client. #4785
 * Install pack dependencies automatically. #4769
-* Add support for `immutable_parameters` on Action Aliases. This feature allows default parameters to be supplied to the action on every execution of the alias. #4786
-* Add ``get_entrypoint()`` method to ``ActionResourceManager`` attribute of st2client. #4791  
+* Add support for `immutable_parameters` on Action Aliases. This feature allows default
+  parameters to be supplied to the action on every execution of the alias. #4786
+* Add ``get_entrypoint()`` method to ``ActionResourceManager`` attribute of st2client.
+  #4791  
 
 Changed
 ~~~~~~~
@@ -34,6 +36,9 @@ Changed
   (improvement)
 
   Reported and contributed by Joshua Meyer (@jdmeyer3) #4803
+* Add new ``action_runner.pip_opts`` st2.conf config option which allows user to specify a list
+  of command line option which are passed to ``pip install`` command when installing pack
+  dependencies into a pack specific virtual environment. #4792
 
 Fixed
 ~~~~~

conf/st2.conf.sample

@@ -8,6 +8,8 @@ enable = True
 emit_when = succeeded,failed,timeout,canceled,abandoned # comma separated list allowed here.
 
 [actionrunner]
+# List of pip options to be passed to "pip install" command when installing pack dependencies into pack virtual environment.
+pip_opts =  # comma separated list allowed here.
 # Internal pool size for dispatcher used by regular actions.
 actions_pool_size = 60
 # Default log level to use for Python runner actions. Can be overriden on invocation basis using "log_level" runner parameter.

st2common/st2common/config.py

@@ -374,6 +374,10 @@ def register_opts(ignore_errors=False):
             'virtualenv_opts', default=['--system-site-packages'],
             help='List of virtualenv options to be passsed to "virtualenv" command that '
                  'creates pack virtualenv.'),
+        cfg.ListOpt(
+            'pip_opts', default=[],
+            help='List of pip options to be passed to "pip install" command when installing pack '
+                 'dependencies into pack virtual environment.'),
         cfg.BoolOpt(
             'stream_output', default=True,
             help='True to store and stream action output (stdout and stderr) in real-time.'),

st2common/st2common/log.py

@@ -181,10 +181,22 @@ def _redirect_stderr():
     sys.stderr = LoggingStream('STDERR')
 
 
-def setup(config_file, redirect_stderr=True, excludes=None, disable_existing_loggers=False):
+def setup(config_file, redirect_stderr=True, excludes=None, disable_existing_loggers=False,
+          st2_conf_path=None):
     """
     Configure logging from file.
+
+    :param st2_conf_path: Optional path to st2.conf file. If provided and "config_file" path is
+                          relative to st2.conf path, the config_file path will get resolved to full
+                          absolute path relative to st2.conf.
+    :type st2_conf_path: ``str``
     """
+    if st2_conf_path and config_file[:2] == './' and not os.path.isfile(config_file):
+        # Logging config path is relative to st2.conf, resolve it to full absolute path
+        directory = os.path.dirname(st2_conf_path)
+        config_file_name = os.path.basename(config_file)
+        config_file = os.path.join(directory, config_file_name)
+
     try:
         logging.config.fileConfig(config_file,
                                   defaults=None,

st2common/st2common/util/concurrency.py

@@ -41,7 +41,9 @@
     'spawn',
     'wait',
     'cancel',
-    'kill'
+    'kill',
+    'sleep',
+    'get_greenlet_exit_exception_class'
 ]
 
 
@@ -111,3 +113,21 @@ def kill(green_thread, *args, **kwargs):
         return green_thread.kill(*args, **kwargs)
     else:
         raise ValueError('Unsupported concurrency library')
+
+
+def sleep(*args, **kwargs):
+    if CONCURRENCY_LIBRARY == 'eventlet':
+        return eventlet.sleep(*args, **kwargs)
+    elif CONCURRENCY_LIBRARY == 'gevent':
+        return gevent.sleep(*args, **kwargs)
+    else:
+        raise ValueError('Unsupported concurrency library')
+
+
+def get_greenlet_exit_exception_class():
+    if CONCURRENCY_LIBRARY == 'eventlet':
+        return eventlet.support.greenlets.GreenletExit
+    elif CONCURRENCY_LIBRARY == 'gevent':
+        return gevent.GreenletExit
+    else:
+        raise ValueError('Unsupported concurrency library')

st2common/st2common/util/virtualenvs.py

@@ -158,7 +158,7 @@ def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_se
     python_binary = cfg.CONF.actionrunner.python_binary
     python3_binary = cfg.CONF.actionrunner.python3_binary
     virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary
-    virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts
+    virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts or []
 
     if not os.path.isfile(python_binary):
         raise Exception('Python binary "%s" doesn\'t exist' % (python_binary))
@@ -237,6 +237,7 @@ def install_requirements(virtualenv_path, requirements_file_path, proxy_config=N
     """
     logger = logger or LOG
     pip_path = os.path.join(virtualenv_path, 'bin/pip')
+    pip_opts = cfg.CONF.actionrunner.pip_opts or []
     cmd = [pip_path]
 
     if proxy_config:
@@ -253,7 +254,10 @@ def install_requirements(virtualenv_path, requirements_file_path, proxy_config=N
         if cert:
             cmd.extend(['--cert', cert])
 
-    cmd.extend(['install', '-U', '-r', requirements_file_path])
+    cmd.append('install')
+    cmd.extend(pip_opts)
+    cmd.extend(['-U', '-r', requirements_file_path])
+
     env = get_env_for_subprocess_command()
 
     logger.debug('Installing requirements from file %s with command %s.',
@@ -278,6 +282,7 @@ def install_requirement(virtualenv_path, requirement, proxy_config=None, logger=
     """
     logger = logger or LOG
     pip_path = os.path.join(virtualenv_path, 'bin/pip')
+    pip_opts = cfg.CONF.actionrunner.pip_opts or []
     cmd = [pip_path]
 
     if proxy_config:
@@ -294,7 +299,9 @@ def install_requirement(virtualenv_path, requirement, proxy_config=None, logger=
         if cert:
             cmd.extend(['--cert', cert])
 
-    cmd.extend(['install', requirement])
+    cmd.append('install')
+    cmd.extend(pip_opts)
+    cmd.extend([requirement])
     env = get_env_for_subprocess_command()
     logger.debug('Installing requirement %s with command %s.',
                  requirement, ' '.join(cmd))

st2reactor/st2reactor/cmd/timersengine.py

@@ -13,13 +13,14 @@
 # limitations under the License.
 
 from __future__ import absolute_import
+
 import os
 import sys
 
-import eventlet
 from oslo_config import cfg
 
 from st2common import log as logging
+from st2common.util import concurrency
 from st2common.constants.timer import TIMER_ENABLED_LOG_LINE, TIMER_DISABLED_LOG_LINE
 from st2common.logging.misc import get_logger_name_for_module
 from st2common.service_setup import setup as common_setup
@@ -61,7 +62,7 @@ def _run_worker():
         if cfg.CONF.timer.enable or cfg.CONF.timersengine.enable:
             local_tz = cfg.CONF.timer.local_timezone or cfg.CONF.timersengine.local_timezone
             timer = St2Timer(local_timezone=local_tz)
-            timer_thread = eventlet.spawn(_kickoff_timer, timer)
+            timer_thread = concurrency.spawn(_kickoff_timer, timer)
             LOG.info(TIMER_ENABLED_LOG_LINE)
             return timer_thread.wait()
         else:
@@ -84,7 +85,7 @@ def main():
         return _run_worker()
     except SystemExit as exit_code:
         sys.exit(exit_code)
-    except:
+    except Exception:
         LOG.exception('(PID=%s) TimerEngine quit due to exception.', os.getpid())
         return 1
     finally:

st2reactor/st2reactor/container/manager.py

@@ -13,13 +13,13 @@
 # limitations under the License.
 
 from __future__ import absolute_import
+
 import os
 import sys
 import signal
 
-import eventlet
-
 from st2common import log as logging
+from st2common.util import concurrency
 from st2reactor.container.process_container import ProcessSensorContainer
 from st2common.services.sensor_watcher import SensorWatcher
 from st2common.models.system.common import ResourceReference
@@ -75,7 +75,7 @@ def _spin_container_and_wait(self, sensors):
             self._sensor_container = ProcessSensorContainer(
                 sensors=sensors,
                 single_sensor_mode=self._single_sensor_mode)
-            self._container_thread = eventlet.spawn(self._sensor_container.run)
+            self._container_thread = concurrency.spawn(self._sensor_container.run)
 
             LOG.debug('Starting sensor CUD watcher...')
             self._sensors_watcher.start()
@@ -90,7 +90,7 @@ def _spin_container_and_wait(self, sensors):
             LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(),
                      sys.exc_info()[0].__name__)
 
-            eventlet.kill(self._container_thread)
+            concurrency.kill(self._container_thread)
             self._container_thread = None
 
             return exit_code