Fix PYTHONPATH handling for Python runner actions using --python3 flag

CHANGELOG.rst

@@ -22,6 +22,13 @@ Fixed
   server time where st2api is running was not set to UTC. (bug fix) #4668
 
   Contributed by Igor Cherkaev. (@emptywee)
+* Fix a bug with some packs which use ``--python3`` flag (running Python 3 actions on installation
+  where StackStorm components run under Python 2) which rely on modules from Python 3 standard
+  library which are also available in Python 2 site-packages (e.g. ``concurrent``) not working
+  correctly.
+
+  In such scenario, package / module was incorrectly loaded from Python 2 site-packages instead of
+  Python 3 standard library which broke such packs. (bug fix) #4658
 
 3.0.0 - April 18, 2019
 ----------------------

contrib/hello_st2/pack.yaml

@@ -1,13 +1,26 @@
 ---
+# Pack reference. It can only contain letters, digits and underscores.
 ref: hello_st2
+# User-friendly pack name. If this attribute contains spaces or any other special characters, then
+# the "ref" attribute must also be specified (see above).
 name: Hello StackStorm
+# User-friendly pack description.
 description: Simple pack containing examples of sensor, rule, and action.
+# Keywords which are used when searching for packs.
 keywords:
     - example
     - test
+# Pack version which must follow semver format (<major>.<minor>.<patch> e.g. 1.0.0)
 version: 0.1.0
+# A list of major Python versions pack is tested with and works with.
 python_versions:
   - "2"
   - "3"
+# Name of the pack author.
 author: StackStorm, Inc.
+# Email of the pack author.
 email: info@stackstorm.com
+# Optional list of additional contributors to the pack.
+contributors:
+  - "John Doe1 <john.doe1@gmail.com>"
+  - "John Doe2 <john.doe2@gmail.com>"

st2common/st2common/util/sandboxing.py

@@ -160,8 +160,30 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True,
         python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path,
                                                        virtualenv_directories[0],
                                                        'site-packages')
-        sandbox_python_path = (python3_lib_directory + ':' + python3_site_packages_directory + ':' +
-                               pack_actions_lib_paths + ':' + sandbox_python_path)
+
+        # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv
+        # one
+        # NOTE: abc.py is always available in base lib directory which is symlinked to virtualenv
+        # lib directory
+        abc_module_path = os.path.join(python3_lib_directory, 'abc.py')
+        link_path = os.path.realpath(abc_module_path)
+        python3_system_lib_directory = os.path.dirname(link_path)
+
+        if not os.path.exists(python3_system_lib_directory):
+            python3_system_lib_directory = None
+
+        full_sandbox_python_path = []
+
+        # NOTE: Order here is very important for imports to function correctly
+        if python3_lib_directory:
+            full_sandbox_python_path.append(python3_system_lib_directory)
+
+        full_sandbox_python_path.append(python3_lib_directory)
+        full_sandbox_python_path.append(python3_site_packages_directory)
+        full_sandbox_python_path.append(pack_actions_lib_paths)
+        full_sandbox_python_path.append(sandbox_python_path)
+
+        sandbox_python_path = ':'.join(full_sandbox_python_path)
 
     return sandbox_python_path
 

st2common/tests/unit/test_util_sandboxing.py

@@ -146,6 +146,7 @@ def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self,
     @mock.patch('os.path.exists', mock.Mock(return_value=True))
     @mock.patch('os.path.isdir', mock.Mock(return_value=True))
     @mock.patch('os.listdir', mock.Mock(return_value=['python3.6']))
+    @mock.patch('os.path.realpath', mock.Mock(return_value='/usr/lib/python3.6/abc.py'))
     @mock.patch('st2common.util.sandboxing.get_python_lib')
     @mock.patch('st2common.util.sandboxing.get_pack_base_path',
                 mock.Mock(return_value='/tmp/packs/dummy_pack'))
@@ -161,16 +162,17 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self,
                                                                 inherit_parent_virtualenv=False)
 
         split = python_path.strip(':').split(':')
-        self.assertEqual(len(split), 3)
+        self.assertEqual(len(split), 4)
 
-        # First entry should be lib/python3 dir from venv
-        self.assertTrue('virtualenvs/dummy_pack/lib/python3.6' in split[0])
+        # First entry should be system lib/python3 dir
+        # Second entry should be lib/python3 dir from venv
+        self.assertTrue('virtualenvs/dummy_pack/lib/python3.6' in split[1])
 
-        # Second entry should be python3 site-packages dir from venv
-        self.assertTrue('virtualenvs/dummy_pack/lib/python3.6/site-packages' in split[1])
+        # Third entry should be python3 site-packages dir from venv
+        self.assertTrue('virtualenvs/dummy_pack/lib/python3.6/site-packages' in split[2])
 
-        # Third entry should be actions/lib dir from pack root directory
-        self.assertTrue('packs/dummy_pack/actions/lib/' in split[2])
+        # Fourth entry should be actions/lib dir from pack root directory
+        self.assertTrue('packs/dummy_pack/actions/lib/' in split[3])
 
         # Inherit python path from current process
         # Mock the current process python path
@@ -179,7 +181,8 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self,
         python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack',
                                                                 inherit_from_parent=True,
                                                                 inherit_parent_virtualenv=False)
-        expected = ('/tmp/virtualenvs/dummy_pack/lib/python3.6:'
+        expected = ('/usr/lib/python3.6:'
+                    '/tmp/virtualenvs/dummy_pack/lib/python3.6:'
                     '/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:'
                     '/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2')
         self.assertEqual(python_path, expected)
@@ -198,7 +201,8 @@ def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self,
                                                                 inherit_from_parent=True,
                                                                 inherit_parent_virtualenv=True)
 
-        expected = ('/tmp/virtualenvs/dummy_pack/lib/python3.6:'
+        expected = ('/usr/lib/python3.6:'
+                    '/tmp/virtualenvs/dummy_pack/lib/python3.6:'
                     '/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:'
                     '/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2:'
                     '%s/virtualenvtest' % (sys.prefix))