From ab7a87f42d2241a4e3dca887a38578ad78505d32 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 8 May 2019 21:13:01 +0200 Subject: [PATCH 1/5] Fix a bug with base system Python 3 lib directory (e.g. /usr/lib/python3.6) not being in the PYTHONPATH for Python actions which used --python3 flag. This meant some packs which rely on 3rd party packages / modules which are available as part of Python 3 standard library and also Python 2 site-packages were incorrectly loaded from Python 2 site-packages instead of Python 3 standard library. --- st2common/st2common/util/sandboxing.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 548c4fff80..33f489ef62 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -160,8 +160,30 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0], 'site-packages') - sandbox_python_path = (python3_lib_directory + ':' + python3_site_packages_directory + ':' + - pack_actions_lib_paths + ':' + sandbox_python_path) + + # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv + # one + # NOTE: abc.py is always available in base lib directory which is symlinked to virtualenv + # lib directory + abc_module_path = os.path.join(python3_lib_directory, 'abc.py') + link_path = os.path.realpath(abc_module_path) + python3_system_lib_directory = os.path.dirname(link_path) + + if not os.path.exists(python3_system_lib_directory): + python3_system_lib_directory = None + + full_sandbox_python_path = [] + + # NOTE: Order here is very important for imports to function correctly + if python3_lib_directory: + full_sandbox_python_path.append(python3_system_lib_directory) + + full_sandbox_python_path.append(python3_lib_directory) + full_sandbox_python_path.append(python3_site_packages_directory) + full_sandbox_python_path.append(pack_actions_lib_paths) + full_sandbox_python_path.append(sandbox_python_path) + + sandbox_python_path= ':'.join(full_sandbox_python_path) return sandbox_python_path From c4cb87a7c9a45e096ff3599999519995d600bd51 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 8 May 2019 21:14:10 +0200 Subject: [PATCH 2/5] Add unit tests for it. --- st2common/tests/unit/test_util_sandboxing.py | 22 ++++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/st2common/tests/unit/test_util_sandboxing.py b/st2common/tests/unit/test_util_sandboxing.py index b9f26d939d..5f4648b7e6 100644 --- a/st2common/tests/unit/test_util_sandboxing.py +++ b/st2common/tests/unit/test_util_sandboxing.py @@ -146,6 +146,7 @@ def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self, @mock.patch('os.path.exists', mock.Mock(return_value=True)) @mock.patch('os.path.isdir', mock.Mock(return_value=True)) @mock.patch('os.listdir', mock.Mock(return_value=['python3.6'])) + @mock.patch('os.path.realpath', mock.Mock(return_value='/usr/lib/python3.6/abc.py')) @mock.patch('st2common.util.sandboxing.get_python_lib') @mock.patch('st2common.util.sandboxing.get_pack_base_path', mock.Mock(return_value='/tmp/packs/dummy_pack')) @@ -161,16 +162,17 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self, inherit_parent_virtualenv=False) split = python_path.strip(':').split(':') - self.assertEqual(len(split), 3) + self.assertEqual(len(split), 4) - # First entry should be lib/python3 dir from venv - self.assertTrue('virtualenvs/dummy_pack/lib/python3.6' in split[0]) + # First entry should be system lib/python3 dir + # Second entry should be lib/python3 dir from venv + self.assertTrue('virtualenvs/dummy_pack/lib/python3.6' in split[1]) - # Second entry should be python3 site-packages dir from venv - self.assertTrue('virtualenvs/dummy_pack/lib/python3.6/site-packages' in split[1]) + # Third entry should be python3 site-packages dir from venv + self.assertTrue('virtualenvs/dummy_pack/lib/python3.6/site-packages' in split[2]) - # Third entry should be actions/lib dir from pack root directory - self.assertTrue('packs/dummy_pack/actions/lib/' in split[2]) + # Fourth entry should be actions/lib dir from pack root directory + self.assertTrue('packs/dummy_pack/actions/lib/' in split[3]) # Inherit python path from current process # Mock the current process python path @@ -179,7 +181,8 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self, python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', inherit_from_parent=True, inherit_parent_virtualenv=False) - expected = ('/tmp/virtualenvs/dummy_pack/lib/python3.6:' + expected = ('/usr/lib/python3.6:' + '/tmp/virtualenvs/dummy_pack/lib/python3.6:' '/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:' '/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2') self.assertEqual(python_path, expected) @@ -198,7 +201,8 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self, inherit_from_parent=True, inherit_parent_virtualenv=True) - expected = ('/tmp/virtualenvs/dummy_pack/lib/python3.6:' + expected = ('/usr/lib/python3.6:' + '/tmp/virtualenvs/dummy_pack/lib/python3.6:' '/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:' '/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2:' '%s/virtualenvtest' % (sys.prefix)) From 346910f83e622fdf8086f55121d5cc0054473da3 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 8 May 2019 21:18:16 +0200 Subject: [PATCH 3/5] Add changelog entry for it. --- CHANGELOG.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8fc416f4c7..5b8fb0e3f4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,13 @@ Fixed In StackStorm v3.0.0, this variable was inadvertently removed from the context used by Mistral workflows. (bug fix) #4650 #4656 +* Fix a bug with some packs which use ``--python3`` flag (running Python 3 actions on installation + where StackStorm components run under Python 2) which rely on modules from Python 3 standard + library which are also available in Python 2 site-packages (e.g. ``concurrent``) not working + correctly. + + In such scenario, package / module was incorrectly loaded from Python 2 site-packages instead of + Python 3 standard library which broke such packs. (bug fix) #4658 3.0.0 - April 18, 2019 ---------------------- From 9d05b4309ec41371df4bea7caedd5c16b2fc5ea0 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 10 May 2019 09:53:10 +0200 Subject: [PATCH 4/5] Fix lint. --- st2common/st2common/util/sandboxing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 33f489ef62..4d172515d0 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -183,7 +183,7 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, full_sandbox_python_path.append(pack_actions_lib_paths) full_sandbox_python_path.append(sandbox_python_path) - sandbox_python_path= ':'.join(full_sandbox_python_path) + sandbox_python_path = ':'.join(full_sandbox_python_path) return sandbox_python_path From bfc0e9302c33faa4198dea383401e223fb3ffbd5 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 10 May 2019 12:38:41 +0200 Subject: [PATCH 5/5] Add some comments to the hello_st2 pack metadata file. --- contrib/hello_st2/pack.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/contrib/hello_st2/pack.yaml b/contrib/hello_st2/pack.yaml index cf2f1ef9a9..365ec13e40 100644 --- a/contrib/hello_st2/pack.yaml +++ b/contrib/hello_st2/pack.yaml @@ -1,13 +1,26 @@ --- +# Pack reference. It can only contain letters, digits and underscores. ref: hello_st2 +# User-friendly pack name. If this attribute contains spaces or any other special characters, then +# the "ref" attribute must also be specified (see above). name: Hello StackStorm +# User-friendly pack description. description: Simple pack containing examples of sensor, rule, and action. +# Keywords which are used when searching for packs. keywords: - example - test +# Pack version which must follow semver format (.. e.g. 1.0.0) version: 0.1.0 +# A list of major Python versions pack is tested with and works with. python_versions: - "2" - "3" +# Name of the pack author. author: StackStorm, Inc. +# Email of the pack author. email: info@stackstorm.com +# Optional list of additional contributors to the pack. +contributors: + - "John Doe1 " + - "John Doe2 "