From d2160d7e2518fb42925747d12269a785b7731e64 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 13 May 2019 10:17:25 +0200 Subject: [PATCH 1/4] Fix the approach we use for determing system lib path for Python 3 virtual environments. Previous approach wasn't robust enough and didn't work on all the distros. --- CHANGELOG.rst | 2 +- st2common/st2common/util/sandboxing.py | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4749109323..5bcdf5b35e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -28,7 +28,7 @@ Fixed correctly. In such scenario, package / module was incorrectly loaded from Python 2 site-packages instead of - Python 3 standard library which broke such packs. (bug fix) #4658 + Python 3 standard library which broke such packs. (bug fix) #4658 #4674 3.0.0 - April 18, 2019 ---------------------- diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 4d172515d0..33f3394295 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -162,20 +162,25 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, 'site-packages') # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv - # one - # NOTE: abc.py is always available in base lib directory which is symlinked to virtualenv - # lib directory - abc_module_path = os.path.join(python3_lib_directory, 'abc.py') - link_path = os.path.realpath(abc_module_path) - python3_system_lib_directory = os.path.dirname(link_path) - - if not os.path.exists(python3_system_lib_directory): + # one (e.g. /usr/lib/python3.6) + # NOTE: We can't simply use sys.prefix dir since it will be set to /opt/stackstorm/st2 + system_prefix_dirs = ['/usr/lib', '/usr/local/lib'] + + # TODO: Support for custom prefix via st2.conf? + for system_prefix_dir in system_prefix_dirs: + python3_system_lib_directory = os.path.join(system_prefix_dir, + virtualenv_directories[0]) + + if os.path.exists(python3_system_lib_directory): + break + + if not python3_system_lib_directory or not os.path.exists(python3_system_lib_directory): python3_system_lib_directory = None full_sandbox_python_path = [] # NOTE: Order here is very important for imports to function correctly - if python3_lib_directory: + if python3_system_lib_directory: full_sandbox_python_path.append(python3_system_lib_directory) full_sandbox_python_path.append(python3_lib_directory) From c5eedf84cd48e88bf0c21196c714b4e84a899e2c Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 13 May 2019 10:24:42 +0200 Subject: [PATCH 2/4] Update affected tests. --- st2common/tests/unit/test_util_sandboxing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/st2common/tests/unit/test_util_sandboxing.py b/st2common/tests/unit/test_util_sandboxing.py index 5f4648b7e6..44a9c5acc6 100644 --- a/st2common/tests/unit/test_util_sandboxing.py +++ b/st2common/tests/unit/test_util_sandboxing.py @@ -146,7 +146,6 @@ def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self, @mock.patch('os.path.exists', mock.Mock(return_value=True)) @mock.patch('os.path.isdir', mock.Mock(return_value=True)) @mock.patch('os.listdir', mock.Mock(return_value=['python3.6'])) - @mock.patch('os.path.realpath', mock.Mock(return_value='/usr/lib/python3.6/abc.py')) @mock.patch('st2common.util.sandboxing.get_python_lib') @mock.patch('st2common.util.sandboxing.get_pack_base_path', mock.Mock(return_value='/tmp/packs/dummy_pack')) From e2420a533298630ab5b5733203e992e7b61deccf Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 13 May 2019 12:44:24 +0200 Subject: [PATCH 3/4] Add support for Python 3 installations which use custom prefix. Prefix can be configured using actionrunner.python3_prefix config option. --- st2common/st2common/config.py | 6 ++++- st2common/st2common/util/sandboxing.py | 11 +++++++-- st2common/tests/unit/test_util_sandboxing.py | 26 ++++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index 9128477865..5d33e440d4 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -358,7 +358,11 @@ def register_opts(ignore_errors=False): cfg.StrOpt( 'python3_binary', default=default_python3_bin_path, help='Python 3 binary which will be used by Python actions for packs which ' - 'use Python 3 virtual environment'), + 'use Python 3 virtual environment.'), + cfg.StrOpt( + 'python3_prefix', default=None, + help='Prefix for Python 3 installation (e.g. /opt/python3.6). If not specified, it ' + 'tries to find Python 3 libraries in /usr/lib and /usr/local/lib.'), cfg.StrOpt( 'virtualenv_binary', default=default_virtualenv_bin_path, help='Virtualenv binary which should be used to create pack virtualenvs.'), diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 33f3394295..60d04f42a7 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -164,9 +164,16 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv # one (e.g. /usr/lib/python3.6) # NOTE: We can't simply use sys.prefix dir since it will be set to /opt/stackstorm/st2 - system_prefix_dirs = ['/usr/lib', '/usr/local/lib'] - # TODO: Support for custom prefix via st2.conf? + system_prefix_dirs = [] + # Take custom prefix into account (if specified) + if cfg.CONF.actionrunner.python3_prefix: + system_prefix_dirs.append(cfg.CONF.actionrunner.python3_prefix) + + # By default, Python libs are installed either in /usr/lib/python3.x or + # /usr/local/lib/python3.x + system_prefix_dirs.extend(['/usr/lib', '/usr/local/lib']) + for system_prefix_dir in system_prefix_dirs: python3_system_lib_directory = os.path.join(system_prefix_dir, virtualenv_directories[0]) diff --git a/st2common/tests/unit/test_util_sandboxing.py b/st2common/tests/unit/test_util_sandboxing.py index 44a9c5acc6..7ed7082949 100644 --- a/st2common/tests/unit/test_util_sandboxing.py +++ b/st2common/tests/unit/test_util_sandboxing.py @@ -164,6 +164,8 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self, self.assertEqual(len(split), 4) # First entry should be system lib/python3 dir + self.assertTrue('/usr/lib/python3.6' in split[0]) + # Second entry should be lib/python3 dir from venv self.assertTrue('virtualenvs/dummy_pack/lib/python3.6' in split[1]) @@ -206,3 +208,27 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self, '/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2:' '%s/virtualenvtest' % (sys.prefix)) self.assertEqual(python_path, expected) + + # Custom prefix specified in the config + cfg.CONF.set_override(name='python3_prefix', override='/opt/lib', + group='actionrunner') + + # No inheritance + python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', + inherit_from_parent=False, + inherit_parent_virtualenv=False) + + split = python_path.strip(':').split(':') + self.assertEqual(len(split), 4) + + # First entry should be system lib/python3 dir + self.assertTrue('/opt/lib/python3.6' in split[0]) + + # Second entry should be lib/python3 dir from venv + self.assertTrue('virtualenvs/dummy_pack/lib/python3.6' in split[1]) + + # Third entry should be python3 site-packages dir from venv + self.assertTrue('virtualenvs/dummy_pack/lib/python3.6/site-packages' in split[2]) + + # Fourth entry should be actions/lib dir from pack root directory + self.assertTrue('packs/dummy_pack/actions/lib/' in split[3]) From 3d33defe9cd50bcd76a833e16d42fd39f245741f Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 13 May 2019 13:19:49 +0200 Subject: [PATCH 4/4] Re-generate sample config. --- conf/st2.conf.sample | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index dada02cf2b..74a47e795d 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -14,16 +14,18 @@ actions_pool_size = 60 python_runner_log_level = DEBUG # Internal pool size for dispatcher used by workflow actions. workflows_pool_size = 40 +# Prefix for Python 3 installation (e.g. /opt/python3.6). If not specified, it tries to find Python 3 libraries in /usr/lib and /usr/local/lib. +python3_prefix = None # Virtualenv binary which should be used to create pack virtualenvs. virtualenv_binary = /usr/bin/virtualenv -# Python 3 binary which will be used by Python actions for packs which use Python 3 virtual environment +# Python 3 binary which will be used by Python actions for packs which use Python 3 virtual environment. python3_binary = /usr/bin/python3 -# location of the logging.conf file -logging = /etc/st2/logging.actionrunner.conf -# True to store and stream action output (stdout and stderr) in real-time. -stream_output = True # List of virtualenv options to be passsed to "virtualenv" command that creates pack virtualenv. virtualenv_opts = --system-site-packages # comma separated list allowed here. +# True to store and stream action output (stdout and stderr) in real-time. +stream_output = True +# location of the logging.conf file +logging = /etc/st2/logging.actionrunner.conf # Python binary which will be used by Python actions. python_binary = /usr/bin/python