Rescue Python path code

st2common/st2common/models/db/execution.py

@@ -97,7 +97,7 @@ class ActionExecutionDB(stormbase.StormFoundationDB):
     }
 
     def get_uid(self):
-        # TODO Construct od from non id field:
+        # TODO Construct id from non id field:
         uid = [self.RESOURCE_TYPE, str(self.id)]  # pylint: disable=no-member
         return ':'.join(uid)
 

st2common/st2common/util/sandboxing.py

@@ -20,13 +20,15 @@
 
 from __future__ import absolute_import
 
+import fnmatch
 import os
 import sys
 from distutils.sysconfig import get_python_lib
 
 from oslo_config import cfg
 
 from st2common.constants.pack import SYSTEM_PACK_NAMES
+from st2common.content.utils import get_pack_base_path
 
 __all__ = [
     'get_sandbox_python_binary_path',
@@ -132,10 +134,45 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True,
     Same as get_sandbox_python_path() function, but it's intended to be used for Python runner
     actions.
     """
-    return get_sandbox_python_path(
+    sandbox_python_path = get_sandbox_python_path(
         inherit_from_parent=inherit_from_parent,
         inherit_parent_virtualenv=inherit_parent_virtualenv)
 
+    pack_base_path = get_pack_base_path(pack_name=pack)
+    virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
+
+    if virtualenv_path and os.path.isdir(virtualenv_path):
+        pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib')
+
+        virtualenv_directories = os.listdir(pack_virtualenv_lib_path)
+        virtualenv_directories = [dir_name for dir_name in virtualenv_directories if
+                                  fnmatch.fnmatch(dir_name, 'python*')]
+
+        # Add the pack's lib directory (lib/python3.x) in front of the PYTHONPATH.
+        pack_actions_lib_paths = os.path.join(pack_base_path, 'actions', 'lib')
+        pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib')
+        pack_venv_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0])
+
+        # Add the pack's site-packages directory (lib/python3.x/site-packages)
+        # in front of the Python system site-packages This is important because
+        # we want Python 3 compatible libraries to be used from the pack virtual
+        # environment and not system ones.
+        pack_venv_site_packages_directory = os.path.join(pack_virtualenv_lib_path,
+                                                         virtualenv_directories[0],
+                                                         'site-packages')
+
+        full_sandbox_python_path = [
+            # NOTE: Order here is very important for imports to function correctly
+            pack_venv_lib_directory,
+            pack_venv_site_packages_directory,
+            pack_actions_lib_paths,
+            sandbox_python_path,
+        ]
+
+        sandbox_python_path = ':'.join(full_sandbox_python_path)
+
+    return sandbox_python_path
+
 
 def get_sandbox_virtualenv_path(pack):
     """

st2common/tests/unit/test_util_sandboxing.py

@@ -38,29 +38,35 @@
 
 
 class SandboxingUtilsTestCase(unittest.TestCase):
+    maxDiff = None
+
     def setUp(self):
         super(SandboxingUtilsTestCase, self).setUp()
 
-        # Restore PATH and other variables before each test case
-        os.environ['PATH'] = self.old_path
-        os.environ['PYTHONPATH'] = self.old_python_path
+        # Restore the virtualenv before each test case
         set_virtualenv_prefix(self.old_virtualenv_prefix)
 
     @classmethod
     def setUpClass(cls):
         tests_config.parse_args()
 
         # Store original values so we can restore them in setUp
-        cls.old_path = os.environ.get('PATH', '')
-        cls.old_python_path = os.environ.get('PYTHONPATH', '')
         cls.old_virtualenv_prefix = get_virtualenv_prefix()
 
     @classmethod
     def tearDownClass(cls):
-        os.environ['PATH'] = cls.old_path
-        os.environ['PYTHONPATH'] = cls.old_python_path
         set_virtualenv_prefix(cls.old_virtualenv_prefix)
 
+    def assertEndsWith(self, string, ending_substr, msg=None):
+        msg = msg or "'{string}'' does not end with '{ending_substr}'"
+        try:
+            assert string.endswith(ending_substr) is True
+        except AssertionError as e:
+            print(dir(e))
+            print(e.args)
+            e.args = (msg.format(string=string, ending_substr=ending_substr),)
+            raise e
+
     def test_get_sandbox_python_binary_path(self):
         # Non-system content pack, should use pack specific virtualenv binary
         result = get_sandbox_python_binary_path(pack='mapack')
@@ -72,12 +78,13 @@ def test_get_sandbox_python_binary_path(self):
         self.assertEqual(result, sys.executable)
 
     def test_get_sandbox_path(self):
+        virtualenv_path = '/home/venv/test'
+
         # Mock the current PATH value
-        os.environ['PATH'] = '/home/path1:/home/path2:/home/path3:'
+        with mock.patch.dict(os.environ, {'PATH': '/home/path1:/home/path2:/home/path3:'}):
+            result = get_sandbox_path(virtualenv_path=virtualenv_path)
 
-        virtualenv_path = '/home/venv/test'
-        result = get_sandbox_path(virtualenv_path=virtualenv_path)
-        self.assertEqual(result, '/home/venv/test/bin/:/home/path1:/home/path2:/home/path3')
+        self.assertEqual(result, f'{virtualenv_path}/bin/:/home/path1:/home/path2:/home/path3')
 
     @mock.patch('st2common.util.sandboxing.get_python_lib')
     def test_get_sandbox_python_path(self, mock_get_python_lib):
@@ -88,59 +95,142 @@ def test_get_sandbox_python_path(self, mock_get_python_lib):
 
         # Inherit python path from current process
         # Mock the current process python path
-        os.environ['PYTHONPATH'] = ':/data/test1:/data/test2'
+        with mock.patch.dict(os.environ, {'PYTHONPATH': ':/data/test1:/data/test2'}):
+            python_path = get_sandbox_python_path(inherit_from_parent=True,
+                                                  inherit_parent_virtualenv=False)
 
-        python_path = get_sandbox_python_path(inherit_from_parent=True,
-                                              inherit_parent_virtualenv=False)
         self.assertEqual(python_path, ':/data/test1:/data/test2')
 
         # Inherit from current process and from virtualenv (not running inside virtualenv)
         clear_virtualenv_prefix()
 
-        python_path = get_sandbox_python_path(inherit_from_parent=True,
-                                              inherit_parent_virtualenv=False)
+        with mock.patch.dict(os.environ, {'PYTHONPATH': ':/data/test1:/data/test2'}):
+            python_path = get_sandbox_python_path(inherit_from_parent=True,
+                                                  inherit_parent_virtualenv=False)
+
         self.assertEqual(python_path, ':/data/test1:/data/test2')
 
         # Inherit from current process and from virtualenv (running inside virtualenv)
         sys.real_prefix = '/usr'
-        mock_get_python_lib.return_value = sys.prefix + '/virtualenvtest'
-        python_path = get_sandbox_python_path(inherit_from_parent=True,
-                                              inherit_parent_virtualenv=True)
-        self.assertEqual(python_path, ':/data/test1:/data/test2:%s/virtualenvtest' %
-                         (sys.prefix))
+        mock_get_python_lib.return_value = f'{sys.prefix}/virtualenvtest'
+
+        with mock.patch.dict(os.environ, {'PYTHONPATH': ':/data/test1:/data/test2'}):
+            python_path = get_sandbox_python_path(inherit_from_parent=True,
+                                                  inherit_parent_virtualenv=True)
+
+        self.assertEqual(python_path, f':/data/test1:/data/test2:{sys.prefix}/virtualenvtest')
 
     @mock.patch('os.path.isdir', mock.Mock(return_value=True))
-    @mock.patch('os.listdir', mock.Mock(return_value=['python2.7']))
+    @mock.patch('os.listdir', mock.Mock(return_value=['python3.6']))
     @mock.patch('st2common.util.sandboxing.get_python_lib')
-    def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self,
+    def test_get_sandbox_python_path_for_python_action_no_inheritance(self,
             mock_get_python_lib):
 
         # No inheritance
         python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack',
                                                                 inherit_from_parent=False,
                                                                 inherit_parent_virtualenv=False)
 
-        self.assertEqual(python_path, ':')
+        actual_path = python_path.strip(':').split(':')
+        self.assertEqual(len(actual_path), 3)
+
+        # First entry should be lib/python3 dir from venv
+        self.assertEndsWith(actual_path[0], 'virtualenvs/dummy_pack/lib/python3.6')
+        # Second entry should be python3 site-packages dir from venv
+        self.assertEndsWith(actual_path[1], 'virtualenvs/dummy_pack/lib/python3.6/site-packages')
+        # Third entry should be actions/lib dir from pack root directory
+        self.assertEndsWith(actual_path[2], 'packs/dummy_pack/actions/lib')
+
+    @mock.patch('os.path.isdir', mock.Mock(return_value=True))
+    @mock.patch('os.listdir', mock.Mock(return_value=['python3.6']))
+    @mock.patch('st2common.util.sandboxing.get_python_lib')
+    def test_get_sandbox_python_path_for_python_action_inherit_from_parent_process_only(self,
+            mock_get_python_lib):
 
         # Inherit python path from current process
         # Mock the current process python path
-        os.environ['PYTHONPATH'] = ':/data/test1:/data/test2'
+        with mock.patch.dict(os.environ, {'PYTHONPATH': ':/data/test1:/data/test2'}):
+            python_path = get_sandbox_python_path(inherit_from_parent=True,
+                                                  inherit_parent_virtualenv=False)
+
+            self.assertEqual(python_path, ':/data/test1:/data/test2')
+
+            python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack',
+                                                                    inherit_from_parent=True,
+                                                                    inherit_parent_virtualenv=False)
+
+        actual_path = python_path.strip(':').split(':')
+        self.assertEqual(len(actual_path), 6)
+
+        # First entry should be lib/python3 dir from venv
+        self.assertEndsWith(actual_path[0], 'virtualenvs/dummy_pack/lib/python3.6')
+        # Second entry should be python3 site-packages dir from venv
+        self.assertEndsWith(actual_path[1], 'virtualenvs/dummy_pack/lib/python3.6/site-packages')
+        # Third entry should be actions/lib dir from pack root directory
+        self.assertEndsWith(actual_path[2], 'packs/dummy_pack/actions/lib')
+        # And the rest of the paths from get_sandbox_python_path
+        self.assertEqual(actual_path[3], '')
+        self.assertEqual(actual_path[4], '/data/test1')
+        self.assertEqual(actual_path[5], '/data/test2')
 
-        python_path = get_sandbox_python_path(inherit_from_parent=True,
-                                              inherit_parent_virtualenv=False)
-        self.assertEqual(python_path, ':/data/test1:/data/test2')
+    @mock.patch('os.path.isdir', mock.Mock(return_value=True))
+    @mock.patch('os.listdir', mock.Mock(return_value=['python3.6']))
+    @mock.patch('st2common.util.sandboxing.get_python_lib')
+    def test_get_sandbox_python_path_for_python_action_inherit_from_parent_process_and_venv(self,
+            mock_get_python_lib):
 
         # Inherit from current process and from virtualenv (not running inside virtualenv)
         clear_virtualenv_prefix()
 
-        python_path = get_sandbox_python_path(inherit_from_parent=True,
-                                              inherit_parent_virtualenv=False)
-        self.assertEqual(python_path, ':/data/test1:/data/test2')
+        # Inherit python path from current process
+        # Mock the current process python path
+        with mock.patch.dict(os.environ, {'PYTHONPATH': ':/data/test1:/data/test2'}):
+            python_path = get_sandbox_python_path(inherit_from_parent=True,
+                                                  inherit_parent_virtualenv=False)
+
+            self.assertEqual(python_path, ':/data/test1:/data/test2')
+
+            python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack',
+                                                                    inherit_from_parent=True,
+                                                                    inherit_parent_virtualenv=True)
+
+        actual_path = python_path.strip(':').split(':')
+        self.assertEqual(len(actual_path), 6)
+
+        # First entry should be lib/python3 dir from venv
+        self.assertEndsWith(actual_path[0], 'virtualenvs/dummy_pack/lib/python3.6')
+        # Second entry should be python3 site-packages dir from venv
+        self.assertEndsWith(actual_path[1], 'virtualenvs/dummy_pack/lib/python3.6/site-packages')
+        # Third entry should be actions/lib dir from pack root directory
+        self.assertEndsWith(actual_path[2], 'packs/dummy_pack/actions/lib')
+        # And the rest of the paths from get_sandbox_python_path
+        self.assertEqual(actual_path[3], '')
+        self.assertEqual(actual_path[4], '/data/test1')
+        self.assertEqual(actual_path[5], '/data/test2')
 
         # Inherit from current process and from virtualenv (running inside virtualenv)
         sys.real_prefix = '/usr'
-        mock_get_python_lib.return_value = sys.prefix + '/virtualenvtest'
-        python_path = get_sandbox_python_path(inherit_from_parent=True,
-                                              inherit_parent_virtualenv=True)
-        self.assertEqual(python_path, ':/data/test1:/data/test2:%s/virtualenvtest' %
-                         (sys.prefix))
+        mock_get_python_lib.return_value = f'{sys.prefix}/virtualenvtest'
+
+        # Inherit python path from current process
+        # Mock the current process python path
+        with mock.patch.dict(os.environ, {'PYTHONPATH': ':/data/test1:/data/test2'}):
+            python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack',
+                                                                    inherit_from_parent=True,
+                                                                    inherit_parent_virtualenv=True)
+
+        actual_path = python_path.strip(':').split(':')
+        self.assertEqual(len(actual_path), 7)
+
+        # First entry should be lib/python3 dir from venv
+        self.assertEndsWith(actual_path[0], 'virtualenvs/dummy_pack/lib/python3.6')
+        # Second entry should be python3 site-packages dir from venv
+        self.assertEndsWith(actual_path[1], 'virtualenvs/dummy_pack/lib/python3.6/site-packages')
+        # Third entry should be actions/lib dir from pack root directory
+        self.assertEndsWith(actual_path[2], 'packs/dummy_pack/actions/lib')
+        # The paths from get_sandbox_python_path
+        self.assertEqual(actual_path[3], '')
+        self.assertEqual(actual_path[4], '/data/test1')
+        self.assertEqual(actual_path[5], '/data/test2')
+        # And the parent virtualenv
+        self.assertEqual(actual_path[6], f'{sys.prefix}/virtualenvtest')