From 687d7da79b8499e7e752767b9fa020a81470db4c Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 7 Aug 2018 12:16:50 +0200 Subject: [PATCH 1/5] Make sure we add lib/python3.x directory in the begining of PYTHONPATH for python runner actions for packs which were created with "use_python3=true" flag. This way we ensure modules and packages from Python3 directory are used before modules and packages from python2.7 site-packages directory. This way we avoid issues such as the one with the enum import. --- st2common/st2common/util/sandboxing.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 8b1f388446..83ad74543b 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -151,14 +151,19 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, uses_python3 = False if uses_python3: + # Add Python 3 lib directory infront of the PYTHONPATH. This way we avoid issues with + # module trying to use packages / modules from Python 2.7 site-packages directory over + # Python 3 one + python3_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0]) + # Add Python 3 lib/site-packages directory infront of the system site packages # This is important because we want Python 3 compatible libraries to be used from # the pack virtual environment and not system ones python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0], 'site-packages') - sandbox_python_path = (python3_site_packages_directory + ':' + pack_actions_lib_paths + - ':' + sandbox_python_path) + sandbox_python_path = (python3_lib_directory + ':' + python3_site_packages_directory + ':' + + pack_actions_lib_paths + ':' + sandbox_python_path) return sandbox_python_path From 27e6ba15797c5387ceb3c107dd8d55d168edf594 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 7 Aug 2018 12:21:11 +0200 Subject: [PATCH 2/5] Add a test case for it. --- st2common/tests/unit/test_util_sandboxing.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/st2common/tests/unit/test_util_sandboxing.py b/st2common/tests/unit/test_util_sandboxing.py index 7e6bf8c42e..97477e235f 100644 --- a/st2common/tests/unit/test_util_sandboxing.py +++ b/st2common/tests/unit/test_util_sandboxing.py @@ -141,13 +141,16 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self, inherit_parent_virtualenv=False) split = python_path.strip(':').split(':') - self.assertEqual(len(split), 2) + self.assertEqual(len(split), 3) - # First entry should be python3 site-packages dir from venv - self.assertTrue('virtualenvs/dummy_pack/lib/python3.6/site-packages' in split[0]) + # First entry should be lib/python3 dir from venv + self.assertTrue('virtualenvs/dummy_pack/lib/python3.6' in split[0]) - # Second entry should be actions/lib dir from pack root directory - self.assertTrue('packs/dummy_pack/actions/lib/' in split[1]) + # Second entry should be python3 site-packages dir from venv + self.assertTrue('virtualenvs/dummy_pack/lib/python3.6/site-packages' in split[1]) + + # Third entry should be actions/lib dir from pack root directory + self.assertTrue('packs/dummy_pack/actions/lib/' in split[2]) # Inherit python path from current process # Mock the current process python path @@ -156,7 +159,8 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self, python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', inherit_from_parent=True, inherit_parent_virtualenv=False) - expected = ('/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:' + expected = ('/tmp/virtualenvs/dummy_pack/lib/python3.6:' + '/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:' '/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2') self.assertEqual(python_path, expected) @@ -174,7 +178,8 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self, inherit_from_parent=True, inherit_parent_virtualenv=True) - expected = ('/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:' + expected = ('/tmp/virtualenvs/dummy_pack/lib/python3.6:' + '/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:' '/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2:' '%s/virtualenvtest' % (sys.prefix)) self.assertEqual(python_path, expected) From fed49ab692b12753928fb5446371a50faceef853 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 7 Aug 2018 12:28:17 +0200 Subject: [PATCH 3/5] Add a changelog entry. --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f4f3c546e0..9c8e36d1d6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -74,6 +74,10 @@ Deprecated Fixed ~~~~~ +* Fix an issue with ``AttributeError: module 'enum' has no attribute 'IntFlag'`` error which would + appear when using Python 3 for a particular pack virtual environment and running on RHEL / + CentOS. (bug fix) #4297 + 2.8.1 - July 18, 2018 --------------------- From d69cba6100acc9b7cd94bd95defddaa396cd966e Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 7 Aug 2018 12:28:53 +0200 Subject: [PATCH 4/5] Wrap lines at 100 characters. --- CHANGELOG.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9c8e36d1d6..86bc2e351b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -64,13 +64,6 @@ Changed * Upgrade ``mongoengine`` (0.15.3) and ``pymongo`` (3.7.1) to the latest stable version. Those changes will allow us to support MongoDB 3.6 in the near future. (improvement) #4292 -Deprecated -~~~~~~~~~~ - -* The CloudSlang runner is now deprecated. In StackStorm 3.1 it will be removed from the core StackStorm codebase. - The runner code will be moved to a separate repository, and no longer maintained by the core StackStorm team. - Users will still be able to install and use this runner, but it will require additional steps to install. - Fixed ~~~~~ @@ -78,6 +71,14 @@ Fixed appear when using Python 3 for a particular pack virtual environment and running on RHEL / CentOS. (bug fix) #4297 +Deprecated +~~~~~~~~~~ + +* The CloudSlang runner is now deprecated. In StackStorm 3.1 it will be removed from the core + StackStorm codebase. The runner code will be moved to a separate repository, and no longer + maintained by the core StackStorm team. Users will still be able to install and use this runner, + but it will require additional steps to install. + 2.8.1 - July 18, 2018 --------------------- From 61e47f26de08cf2fb32f0da05dd29e17e31928cc Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 7 Aug 2018 12:34:13 +0200 Subject: [PATCH 5/5] Update comments. --- st2common/st2common/util/sandboxing.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 83ad74543b..eea7fc8cbe 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -151,14 +151,14 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, uses_python3 = False if uses_python3: - # Add Python 3 lib directory infront of the PYTHONPATH. This way we avoid issues with - # module trying to use packages / modules from Python 2.7 site-packages directory over - # Python 3 one + # Add Python 3 lib directory (lib/python3.x) in front of the PYTHONPATH. This way we avoid + # issues with scripts trying to use packages / modules from Python 2.7 site-packages + # directory instead of the versions from Python 3 stdlib. python3_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0]) - # Add Python 3 lib/site-packages directory infront of the system site packages - # This is important because we want Python 3 compatible libraries to be used from - # the pack virtual environment and not system ones + # Add Python 3 site-packages directory (lib/python3.x/site-packages) in front of the Python + # 2.7 system site-packages This is important because we want Python 3 compatible libraries + # to be used from the pack virtual environment and not system ones. python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0], 'site-packages')