diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fba5d936b5..4749109323 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,6 +22,13 @@ Fixed server time where st2api is running was not set to UTC. (bug fix) #4668 Contributed by Igor Cherkaev. (@emptywee) +* Fix a bug with some packs which use ``--python3`` flag (running Python 3 actions on installation + where StackStorm components run under Python 2) which rely on modules from Python 3 standard + library which are also available in Python 2 site-packages (e.g. ``concurrent``) not working + correctly. + + In such scenario, package / module was incorrectly loaded from Python 2 site-packages instead of + Python 3 standard library which broke such packs. (bug fix) #4658 3.0.0 - April 18, 2019 ---------------------- diff --git a/contrib/hello_st2/pack.yaml b/contrib/hello_st2/pack.yaml index cf2f1ef9a9..365ec13e40 100644 --- a/contrib/hello_st2/pack.yaml +++ b/contrib/hello_st2/pack.yaml @@ -1,13 +1,26 @@ --- +# Pack reference. It can only contain letters, digits and underscores. ref: hello_st2 +# User-friendly pack name. If this attribute contains spaces or any other special characters, then +# the "ref" attribute must also be specified (see above). name: Hello StackStorm +# User-friendly pack description. description: Simple pack containing examples of sensor, rule, and action. +# Keywords which are used when searching for packs. keywords: - example - test +# Pack version which must follow semver format (.. e.g. 1.0.0) version: 0.1.0 +# A list of major Python versions pack is tested with and works with. python_versions: - "2" - "3" +# Name of the pack author. author: StackStorm, Inc. +# Email of the pack author. email: info@stackstorm.com +# Optional list of additional contributors to the pack. +contributors: + - "John Doe1 " + - "John Doe2 " diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 548c4fff80..4d172515d0 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -160,8 +160,30 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0], 'site-packages') - sandbox_python_path = (python3_lib_directory + ':' + python3_site_packages_directory + ':' + - pack_actions_lib_paths + ':' + sandbox_python_path) + + # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv + # one + # NOTE: abc.py is always available in base lib directory which is symlinked to virtualenv + # lib directory + abc_module_path = os.path.join(python3_lib_directory, 'abc.py') + link_path = os.path.realpath(abc_module_path) + python3_system_lib_directory = os.path.dirname(link_path) + + if not os.path.exists(python3_system_lib_directory): + python3_system_lib_directory = None + + full_sandbox_python_path = [] + + # NOTE: Order here is very important for imports to function correctly + if python3_lib_directory: + full_sandbox_python_path.append(python3_system_lib_directory) + + full_sandbox_python_path.append(python3_lib_directory) + full_sandbox_python_path.append(python3_site_packages_directory) + full_sandbox_python_path.append(pack_actions_lib_paths) + full_sandbox_python_path.append(sandbox_python_path) + + sandbox_python_path = ':'.join(full_sandbox_python_path) return sandbox_python_path diff --git a/st2common/tests/unit/test_util_sandboxing.py b/st2common/tests/unit/test_util_sandboxing.py index b9f26d939d..5f4648b7e6 100644 --- a/st2common/tests/unit/test_util_sandboxing.py +++ b/st2common/tests/unit/test_util_sandboxing.py @@ -146,6 +146,7 @@ def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self, @mock.patch('os.path.exists', mock.Mock(return_value=True)) @mock.patch('os.path.isdir', mock.Mock(return_value=True)) @mock.patch('os.listdir', mock.Mock(return_value=['python3.6'])) + @mock.patch('os.path.realpath', mock.Mock(return_value='/usr/lib/python3.6/abc.py')) @mock.patch('st2common.util.sandboxing.get_python_lib') @mock.patch('st2common.util.sandboxing.get_pack_base_path', mock.Mock(return_value='/tmp/packs/dummy_pack')) @@ -161,16 +162,17 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self, inherit_parent_virtualenv=False) split = python_path.strip(':').split(':') - self.assertEqual(len(split), 3) + self.assertEqual(len(split), 4) - # First entry should be lib/python3 dir from venv - self.assertTrue('virtualenvs/dummy_pack/lib/python3.6' in split[0]) + # First entry should be system lib/python3 dir + # Second entry should be lib/python3 dir from venv + self.assertTrue('virtualenvs/dummy_pack/lib/python3.6' in split[1]) - # Second entry should be python3 site-packages dir from venv - self.assertTrue('virtualenvs/dummy_pack/lib/python3.6/site-packages' in split[1]) + # Third entry should be python3 site-packages dir from venv + self.assertTrue('virtualenvs/dummy_pack/lib/python3.6/site-packages' in split[2]) - # Third entry should be actions/lib dir from pack root directory - self.assertTrue('packs/dummy_pack/actions/lib/' in split[2]) + # Fourth entry should be actions/lib dir from pack root directory + self.assertTrue('packs/dummy_pack/actions/lib/' in split[3]) # Inherit python path from current process # Mock the current process python path @@ -179,7 +181,8 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self, python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', inherit_from_parent=True, inherit_parent_virtualenv=False) - expected = ('/tmp/virtualenvs/dummy_pack/lib/python3.6:' + expected = ('/usr/lib/python3.6:' + '/tmp/virtualenvs/dummy_pack/lib/python3.6:' '/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:' '/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2') self.assertEqual(python_path, expected) @@ -198,7 +201,8 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self, inherit_from_parent=True, inherit_parent_virtualenv=True) - expected = ('/tmp/virtualenvs/dummy_pack/lib/python3.6:' + expected = ('/usr/lib/python3.6:' + '/tmp/virtualenvs/dummy_pack/lib/python3.6:' '/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:' '/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2:' '%s/virtualenvtest' % (sys.prefix))