From bec90832d7861f2e8b640477873fabcf483dc505 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 26 Jan 2021 14:04:22 -0700 Subject: [PATCH 1/9] Fix implicit relative imports in pack actions --- st2common/st2common/util/sandboxing.py | 53 +++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 6b3c4e29b9..32a3033647 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -20,6 +20,7 @@ from __future__ import absolute_import +import glob import os import sys from distutils.sysconfig import get_python_lib @@ -27,6 +28,7 @@ from oslo_config import cfg from st2common.constants.pack import SYSTEM_PACK_NAMES +from st2common.content.utils import get_pack_base_path __all__ = [ 'get_sandbox_python_binary_path', @@ -132,10 +134,59 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, Same as get_sandbox_python_path() function, but it's intended to be used for Python runner actions. """ - return get_sandbox_python_path( + sandbox_python_path = get_sandbox_python_path( inherit_from_parent=inherit_from_parent, inherit_parent_virtualenv=inherit_parent_virtualenv) + virtualenv_path = get_sandbox_virtualenv_path(pack=pack) + + if virtualenv_path: + pack_base_path = get_pack_base_path(pack_name=pack) + + # Get the pack's actions/lib directory + pack_actions_lib_paths = os.path.join(pack_base_path, 'actions', 'lib') + + # Get the pack's virtualenv site-packages directory + virtualenv_lib_path = os.path.join(virtualenv_path, 'lib', 'python*', 'site-packages') + virtualenv_lib_directory = glob.glob(virtualenv_lib_path)[0] + + # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv + # one (e.g. /usr/lib/python3.6) + # NOTE: We can't simply use sys.prefix dir since it will be set to /opt/stackstorm/st2 + + system_prefix_dirs = [] + # Take custom prefix into account (if specified) + if cfg.CONF.actionrunner.python3_prefix: + system_prefix_dirs.append(cfg.CONF.actionrunner.python3_prefix) + + # By default, Python libs are installed either in /usr/lib/python3.x or + # /usr/local/lib/python3.x + system_prefix_dirs.extend(['/usr/lib', '/usr/local/lib']) + + for system_prefix_dir in system_prefix_dirs: + python3_system_lib_directory = os.path.join(system_prefix_dir, + virtualenv_directories[0]) + + if os.path.exists(python3_system_lib_directory): + break + else: + # If no Python 3 system library is found + python3_system_lib_directory = None + + full_sandbox_python_path = [] + + # NOTE: Order here is very important for imports to function correctly + if python3_system_lib_directory: + full_sandbox_python_path.append(python3_system_lib_directory) + + full_sandbox_python_path.append(virtualenv_lib_directory) + full_sandbox_python_path.append(pack_actions_lib_paths) + full_sandbox_python_path.append(sandbox_python_path) + + sandbox_python_path = ':'.join(full_sandbox_python_path) + + return sandbox_python_path + def get_sandbox_virtualenv_path(pack): """ From a991157e4273b05126a8a937fef41ad58b29e38a Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 26 Jan 2021 14:12:00 -0700 Subject: [PATCH 2/9] Update changelog --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 94db66b6db..e604640fc8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -53,6 +53,10 @@ Fixed * StackStorm now explicitly decodes pack files as utf-8 instead of implicitly as ascii (bug fix) #5106, #5107 +* Kept functionality for implicit relative imports in Python actions (regression, bug fix) #5127 + + Identified by @amanda11 + Removed ~~~~~~~~ * Removed --python3 pack install option #5100 From c1f6c6478db67912a8bcdd0e2bd3d53e836d4270 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 26 Jan 2021 14:15:55 -0700 Subject: [PATCH 3/9] Fix variable reference --- st2common/st2common/util/sandboxing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 32a3033647..c549101012 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -165,7 +165,7 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, for system_prefix_dir in system_prefix_dirs: python3_system_lib_directory = os.path.join(system_prefix_dir, - virtualenv_directories[0]) + virtualenv_lib_directory) if os.path.exists(python3_system_lib_directory): break From c6ab8eb83f6e49821ed693d31b3cc769ecea80fb Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 26 Jan 2021 14:34:28 -0700 Subject: [PATCH 4/9] Handle a non-existing virtualenv --- st2common/st2common/util/sandboxing.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index c549101012..165a4467dc 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -140,15 +140,19 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, virtualenv_path = get_sandbox_virtualenv_path(pack=pack) - if virtualenv_path: + # Get the pack's virtualenv site-packages directory + # There should only be one, but we don't know what version of Python it + # will be using, so we use a glob and ensure it exists + virtualenv_lib_path = os.path.join(virtualenv_path, 'lib', 'python*', 'site-packages') + virtualenv_lib_directories = glob.glob(virtualenv_lib_path) + + if virtualenv_path and virtualenv_lib_directories: pack_base_path = get_pack_base_path(pack_name=pack) # Get the pack's actions/lib directory pack_actions_lib_paths = os.path.join(pack_base_path, 'actions', 'lib') - - # Get the pack's virtualenv site-packages directory - virtualenv_lib_path = os.path.join(virtualenv_path, 'lib', 'python*', 'site-packages') - virtualenv_lib_directory = glob.glob(virtualenv_lib_path)[0] + # Get the pack's virtualenv's site-packages directory + virtualenv_lib_directory = virtualenv_lib_directories[0] # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv # one (e.g. /usr/lib/python3.6) From 3124596be7dfd5615064dcb39ea569f95fefdcf3 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 26 Jan 2021 14:38:01 -0700 Subject: [PATCH 5/9] Remove python3_prefix handling since we removed that config option --- st2common/st2common/util/sandboxing.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 165a4467dc..0509bffedd 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -158,16 +158,9 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, # one (e.g. /usr/lib/python3.6) # NOTE: We can't simply use sys.prefix dir since it will be set to /opt/stackstorm/st2 - system_prefix_dirs = [] - # Take custom prefix into account (if specified) - if cfg.CONF.actionrunner.python3_prefix: - system_prefix_dirs.append(cfg.CONF.actionrunner.python3_prefix) - # By default, Python libs are installed either in /usr/lib/python3.x or # /usr/local/lib/python3.x - system_prefix_dirs.extend(['/usr/lib', '/usr/local/lib']) - - for system_prefix_dir in system_prefix_dirs: + for system_prefix_dir in ['/usr/lib', '/usr/local/lib']: python3_system_lib_directory = os.path.join(system_prefix_dir, virtualenv_lib_directory) From 9b6e7484f7a45c0415112e60fa0021c2de740e06 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 26 Jan 2021 15:42:22 -0700 Subject: [PATCH 6/9] Refactor a bit --- st2common/st2common/util/sandboxing.py | 95 ++++++++++++++------------ 1 file changed, 53 insertions(+), 42 deletions(-) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 0509bffedd..38e77c3296 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -20,7 +20,7 @@ from __future__ import absolute_import -import glob +import fnmatch import os import sys from distutils.sysconfig import get_python_lib @@ -140,47 +140,58 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, virtualenv_path = get_sandbox_virtualenv_path(pack=pack) - # Get the pack's virtualenv site-packages directory - # There should only be one, but we don't know what version of Python it - # will be using, so we use a glob and ensure it exists - virtualenv_lib_path = os.path.join(virtualenv_path, 'lib', 'python*', 'site-packages') - virtualenv_lib_directories = glob.glob(virtualenv_lib_path) - - if virtualenv_path and virtualenv_lib_directories: - pack_base_path = get_pack_base_path(pack_name=pack) - - # Get the pack's actions/lib directory - pack_actions_lib_paths = os.path.join(pack_base_path, 'actions', 'lib') - # Get the pack's virtualenv's site-packages directory - virtualenv_lib_directory = virtualenv_lib_directories[0] - - # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv - # one (e.g. /usr/lib/python3.6) - # NOTE: We can't simply use sys.prefix dir since it will be set to /opt/stackstorm/st2 - - # By default, Python libs are installed either in /usr/lib/python3.x or - # /usr/local/lib/python3.x - for system_prefix_dir in ['/usr/lib', '/usr/local/lib']: - python3_system_lib_directory = os.path.join(system_prefix_dir, - virtualenv_lib_directory) - - if os.path.exists(python3_system_lib_directory): - break - else: - # If no Python 3 system library is found - python3_system_lib_directory = None - - full_sandbox_python_path = [] - - # NOTE: Order here is very important for imports to function correctly - if python3_system_lib_directory: - full_sandbox_python_path.append(python3_system_lib_directory) - - full_sandbox_python_path.append(virtualenv_lib_directory) - full_sandbox_python_path.append(pack_actions_lib_paths) - full_sandbox_python_path.append(sandbox_python_path) - - sandbox_python_path = ':'.join(full_sandbox_python_path) + if virtualenv_path and os.path.isdir(virtualenv_path): + # Get the pack's virtualenv site-packages directory + # There should only be one, but we don't know what version of Python it + # will be using, so we use a glob and ensure it exists + virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') + virtualenv_lib_directories = [ + dir_name for dir_name in os.listdir(virtualenv_lib_path) + if fnmatch.fnmatch(dir_name, 'python*') + ] + + if virtualenv_lib_directories: + pack_base_path = get_pack_base_path(pack_name=pack) + # This will be just the virtualenv's Python version - eg: python3.6 + virtualenv_lib_directory = virtualenv_lib_directories[0] + + # Get the pack's actions/lib directory + pack_actions_lib_paths = os.path.join(pack_base_path, 'actions', 'lib') + # Get the pack's virtualenv's site-packages directory + virtualenv_lib_directory = os.path.join(virtualenv_lib_path, virtualenv_lib_directory) + + # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv + # one (e.g. /usr/lib/python3.6) + # This code will pick the same sytem Python version as exists in + # the virtualenv, since virtualenv_lib_directory identifies that + # directory within the virtualenv and it should have the same name + # as the system Python executable. + + # NOTE: We can't simply use sys.prefix dir since it will be set to /opt/stackstorm/st2 + + # By default, Python libs are installed either in /usr/lib/python3.x or + # /usr/local/lib/python3.x + for system_prefix_dir in ['/usr/lib', '/usr/local/lib']: + system_lib_directory = os.path.join(system_prefix_dir, + virtualenv_lib_directory) + + if os.path.exists(system_lib_directory): + break + else: + # If no system library is found + system_lib_directory = None + + full_sandbox_python_path = [] + + # NOTE: Order here is very important for imports to function correctly + if python3_system_lib_directory: + full_sandbox_python_path.append(system_lib_directory) + + full_sandbox_python_path.append(virtualenv_lib_directory) + full_sandbox_python_path.append(pack_actions_lib_paths) + full_sandbox_python_path.append(sandbox_python_path) + + sandbox_python_path = ':'.join(full_sandbox_python_path) return sandbox_python_path From aa603c3f27a58a2998dc52298a48ed2d78117dc8 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 26 Jan 2021 15:49:23 -0700 Subject: [PATCH 7/9] Fix unchanged variable name change --- st2common/st2common/util/sandboxing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 38e77c3296..ca26a3e0f7 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -184,7 +184,7 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, full_sandbox_python_path = [] # NOTE: Order here is very important for imports to function correctly - if python3_system_lib_directory: + if system_lib_directory: full_sandbox_python_path.append(system_lib_directory) full_sandbox_python_path.append(virtualenv_lib_directory) From d7d95b94d4d52e5b8ada8564f3f6f22269f4496a Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 26 Jan 2021 17:06:40 -0700 Subject: [PATCH 8/9] Reformat comment --- st2common/st2common/util/sandboxing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index ca26a3e0f7..b6b629a72e 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -160,8 +160,8 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, # Get the pack's virtualenv's site-packages directory virtualenv_lib_directory = os.path.join(virtualenv_lib_path, virtualenv_lib_directory) - # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv - # one (e.g. /usr/lib/python3.6) + # Work around to make sure we also add system lib dir to PYTHONPATH + # and not just virtualenv one (e.g. /usr/lib/python3.6) # This code will pick the same sytem Python version as exists in # the virtualenv, since virtualenv_lib_directory identifies that # directory within the virtualenv and it should have the same name From 4dd202b3cc1c5e574d7aa44a3794c8686e7691e0 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 26 Jan 2021 17:08:52 -0700 Subject: [PATCH 9/9] Mock get_sandbox_virtualenv_path to return None --- st2common/tests/unit/test_util_sandboxing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/st2common/tests/unit/test_util_sandboxing.py b/st2common/tests/unit/test_util_sandboxing.py index af3c4328da..e1da46e257 100644 --- a/st2common/tests/unit/test_util_sandboxing.py +++ b/st2common/tests/unit/test_util_sandboxing.py @@ -111,6 +111,8 @@ def test_get_sandbox_python_path(self, mock_get_python_lib): @mock.patch('os.path.isdir', mock.Mock(return_value=True)) @mock.patch('os.listdir', mock.Mock(return_value=['python2.7'])) + @mock.patch('st2common.util.sandboxing.get_sandbox_virtualenv_path', + mock.Mock(return_value=None)) @mock.patch('st2common.util.sandboxing.get_python_lib') def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self, mock_get_python_lib):