diff --git a/st2common/st2common/models/db/execution.py b/st2common/st2common/models/db/execution.py index d76f583cef..3e8f3c7742 100644 --- a/st2common/st2common/models/db/execution.py +++ b/st2common/st2common/models/db/execution.py @@ -97,7 +97,7 @@ class ActionExecutionDB(stormbase.StormFoundationDB): } def get_uid(self): - # TODO Construct od from non id field: + # TODO Construct id from non id field: uid = [self.RESOURCE_TYPE, str(self.id)] # pylint: disable=no-member return ':'.join(uid) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 6b3c4e29b9..02871f7472 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -20,6 +20,7 @@ from __future__ import absolute_import +import fnmatch import os import sys from distutils.sysconfig import get_python_lib @@ -27,6 +28,7 @@ from oslo_config import cfg from st2common.constants.pack import SYSTEM_PACK_NAMES +from st2common.content.utils import get_pack_base_path __all__ = [ 'get_sandbox_python_binary_path', @@ -132,10 +134,45 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, Same as get_sandbox_python_path() function, but it's intended to be used for Python runner actions. """ - return get_sandbox_python_path( + sandbox_python_path = get_sandbox_python_path( inherit_from_parent=inherit_from_parent, inherit_parent_virtualenv=inherit_parent_virtualenv) + pack_base_path = get_pack_base_path(pack_name=pack) + virtualenv_path = get_sandbox_virtualenv_path(pack=pack) + + if virtualenv_path and os.path.isdir(virtualenv_path): + pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') + + virtualenv_directories = os.listdir(pack_virtualenv_lib_path) + virtualenv_directories = [dir_name for dir_name in virtualenv_directories if + fnmatch.fnmatch(dir_name, 'python*')] + + # Add the pack's lib directory (lib/python3.x) in front of the PYTHONPATH. + pack_actions_lib_paths = os.path.join(pack_base_path, 'actions', 'lib') + pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') + pack_venv_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0]) + + # Add the pack's site-packages directory (lib/python3.x/site-packages) + # in front of the Python system site-packages This is important because + # we want Python 3 compatible libraries to be used from the pack virtual + # environment and not system ones. + pack_venv_site_packages_directory = os.path.join(pack_virtualenv_lib_path, + virtualenv_directories[0], + 'site-packages') + + full_sandbox_python_path = [ + # NOTE: Order here is very important for imports to function correctly + pack_venv_lib_directory, + pack_venv_site_packages_directory, + pack_actions_lib_paths, + sandbox_python_path, + ] + + sandbox_python_path = ':'.join(full_sandbox_python_path) + + return sandbox_python_path + def get_sandbox_virtualenv_path(pack): """ diff --git a/st2common/tests/unit/test_util_sandboxing.py b/st2common/tests/unit/test_util_sandboxing.py index af3c4328da..5f387e0067 100644 --- a/st2common/tests/unit/test_util_sandboxing.py +++ b/st2common/tests/unit/test_util_sandboxing.py @@ -38,12 +38,12 @@ class SandboxingUtilsTestCase(unittest.TestCase): + maxDiff = None + def setUp(self): super(SandboxingUtilsTestCase, self).setUp() - # Restore PATH and other variables before each test case - os.environ['PATH'] = self.old_path - os.environ['PYTHONPATH'] = self.old_python_path + # Restore the virtualenv before each test case set_virtualenv_prefix(self.old_virtualenv_prefix) @classmethod @@ -51,16 +51,22 @@ def setUpClass(cls): tests_config.parse_args() # Store original values so we can restore them in setUp - cls.old_path = os.environ.get('PATH', '') - cls.old_python_path = os.environ.get('PYTHONPATH', '') cls.old_virtualenv_prefix = get_virtualenv_prefix() @classmethod def tearDownClass(cls): - os.environ['PATH'] = cls.old_path - os.environ['PYTHONPATH'] = cls.old_python_path set_virtualenv_prefix(cls.old_virtualenv_prefix) + def assertEndsWith(self, string, ending_substr, msg=None): + msg = msg or "'{string}'' does not end with '{ending_substr}'" + try: + assert string.endswith(ending_substr) is True + except AssertionError as e: + print(dir(e)) + print(e.args) + e.args = (msg.format(string=string, ending_substr=ending_substr),) + raise e + def test_get_sandbox_python_binary_path(self): # Non-system content pack, should use pack specific virtualenv binary result = get_sandbox_python_binary_path(pack='mapack') @@ -72,12 +78,13 @@ def test_get_sandbox_python_binary_path(self): self.assertEqual(result, sys.executable) def test_get_sandbox_path(self): + virtualenv_path = '/home/venv/test' + # Mock the current PATH value - os.environ['PATH'] = '/home/path1:/home/path2:/home/path3:' + with mock.patch.dict(os.environ, {'PATH': '/home/path1:/home/path2:/home/path3:'}): + result = get_sandbox_path(virtualenv_path=virtualenv_path) - virtualenv_path = '/home/venv/test' - result = get_sandbox_path(virtualenv_path=virtualenv_path) - self.assertEqual(result, '/home/venv/test/bin/:/home/path1:/home/path2:/home/path3') + self.assertEqual(result, f'{virtualenv_path}/bin/:/home/path1:/home/path2:/home/path3') @mock.patch('st2common.util.sandboxing.get_python_lib') def test_get_sandbox_python_path(self, mock_get_python_lib): @@ -88,31 +95,35 @@ def test_get_sandbox_python_path(self, mock_get_python_lib): # Inherit python path from current process # Mock the current process python path - os.environ['PYTHONPATH'] = ':/data/test1:/data/test2' + with mock.patch.dict(os.environ, {'PYTHONPATH': ':/data/test1:/data/test2'}): + python_path = get_sandbox_python_path(inherit_from_parent=True, + inherit_parent_virtualenv=False) - python_path = get_sandbox_python_path(inherit_from_parent=True, - inherit_parent_virtualenv=False) self.assertEqual(python_path, ':/data/test1:/data/test2') # Inherit from current process and from virtualenv (not running inside virtualenv) clear_virtualenv_prefix() - python_path = get_sandbox_python_path(inherit_from_parent=True, - inherit_parent_virtualenv=False) + with mock.patch.dict(os.environ, {'PYTHONPATH': ':/data/test1:/data/test2'}): + python_path = get_sandbox_python_path(inherit_from_parent=True, + inherit_parent_virtualenv=False) + self.assertEqual(python_path, ':/data/test1:/data/test2') # Inherit from current process and from virtualenv (running inside virtualenv) sys.real_prefix = '/usr' - mock_get_python_lib.return_value = sys.prefix + '/virtualenvtest' - python_path = get_sandbox_python_path(inherit_from_parent=True, - inherit_parent_virtualenv=True) - self.assertEqual(python_path, ':/data/test1:/data/test2:%s/virtualenvtest' % - (sys.prefix)) + mock_get_python_lib.return_value = f'{sys.prefix}/virtualenvtest' + + with mock.patch.dict(os.environ, {'PYTHONPATH': ':/data/test1:/data/test2'}): + python_path = get_sandbox_python_path(inherit_from_parent=True, + inherit_parent_virtualenv=True) + + self.assertEqual(python_path, f':/data/test1:/data/test2:{sys.prefix}/virtualenvtest') @mock.patch('os.path.isdir', mock.Mock(return_value=True)) - @mock.patch('os.listdir', mock.Mock(return_value=['python2.7'])) + @mock.patch('os.listdir', mock.Mock(return_value=['python3.6'])) @mock.patch('st2common.util.sandboxing.get_python_lib') - def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self, + def test_get_sandbox_python_path_for_python_action_no_inheritance(self, mock_get_python_lib): # No inheritance @@ -120,27 +131,106 @@ def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self, inherit_from_parent=False, inherit_parent_virtualenv=False) - self.assertEqual(python_path, ':') + actual_path = python_path.strip(':').split(':') + self.assertEqual(len(actual_path), 3) + + # First entry should be lib/python3 dir from venv + self.assertEndsWith(actual_path[0], 'virtualenvs/dummy_pack/lib/python3.6') + # Second entry should be python3 site-packages dir from venv + self.assertEndsWith(actual_path[1], 'virtualenvs/dummy_pack/lib/python3.6/site-packages') + # Third entry should be actions/lib dir from pack root directory + self.assertEndsWith(actual_path[2], 'packs/dummy_pack/actions/lib') + + @mock.patch('os.path.isdir', mock.Mock(return_value=True)) + @mock.patch('os.listdir', mock.Mock(return_value=['python3.6'])) + @mock.patch('st2common.util.sandboxing.get_python_lib') + def test_get_sandbox_python_path_for_python_action_inherit_from_parent_process_only(self, + mock_get_python_lib): # Inherit python path from current process # Mock the current process python path - os.environ['PYTHONPATH'] = ':/data/test1:/data/test2' + with mock.patch.dict(os.environ, {'PYTHONPATH': ':/data/test1:/data/test2'}): + python_path = get_sandbox_python_path(inherit_from_parent=True, + inherit_parent_virtualenv=False) + + self.assertEqual(python_path, ':/data/test1:/data/test2') + + python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', + inherit_from_parent=True, + inherit_parent_virtualenv=False) + + actual_path = python_path.strip(':').split(':') + self.assertEqual(len(actual_path), 6) + + # First entry should be lib/python3 dir from venv + self.assertEndsWith(actual_path[0], 'virtualenvs/dummy_pack/lib/python3.6') + # Second entry should be python3 site-packages dir from venv + self.assertEndsWith(actual_path[1], 'virtualenvs/dummy_pack/lib/python3.6/site-packages') + # Third entry should be actions/lib dir from pack root directory + self.assertEndsWith(actual_path[2], 'packs/dummy_pack/actions/lib') + # And the rest of the paths from get_sandbox_python_path + self.assertEqual(actual_path[3], '') + self.assertEqual(actual_path[4], '/data/test1') + self.assertEqual(actual_path[5], '/data/test2') - python_path = get_sandbox_python_path(inherit_from_parent=True, - inherit_parent_virtualenv=False) - self.assertEqual(python_path, ':/data/test1:/data/test2') + @mock.patch('os.path.isdir', mock.Mock(return_value=True)) + @mock.patch('os.listdir', mock.Mock(return_value=['python3.6'])) + @mock.patch('st2common.util.sandboxing.get_python_lib') + def test_get_sandbox_python_path_for_python_action_inherit_from_parent_process_and_venv(self, + mock_get_python_lib): # Inherit from current process and from virtualenv (not running inside virtualenv) clear_virtualenv_prefix() - python_path = get_sandbox_python_path(inherit_from_parent=True, - inherit_parent_virtualenv=False) - self.assertEqual(python_path, ':/data/test1:/data/test2') + # Inherit python path from current process + # Mock the current process python path + with mock.patch.dict(os.environ, {'PYTHONPATH': ':/data/test1:/data/test2'}): + python_path = get_sandbox_python_path(inherit_from_parent=True, + inherit_parent_virtualenv=False) + + self.assertEqual(python_path, ':/data/test1:/data/test2') + + python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', + inherit_from_parent=True, + inherit_parent_virtualenv=True) + + actual_path = python_path.strip(':').split(':') + self.assertEqual(len(actual_path), 6) + + # First entry should be lib/python3 dir from venv + self.assertEndsWith(actual_path[0], 'virtualenvs/dummy_pack/lib/python3.6') + # Second entry should be python3 site-packages dir from venv + self.assertEndsWith(actual_path[1], 'virtualenvs/dummy_pack/lib/python3.6/site-packages') + # Third entry should be actions/lib dir from pack root directory + self.assertEndsWith(actual_path[2], 'packs/dummy_pack/actions/lib') + # And the rest of the paths from get_sandbox_python_path + self.assertEqual(actual_path[3], '') + self.assertEqual(actual_path[4], '/data/test1') + self.assertEqual(actual_path[5], '/data/test2') # Inherit from current process and from virtualenv (running inside virtualenv) sys.real_prefix = '/usr' - mock_get_python_lib.return_value = sys.prefix + '/virtualenvtest' - python_path = get_sandbox_python_path(inherit_from_parent=True, - inherit_parent_virtualenv=True) - self.assertEqual(python_path, ':/data/test1:/data/test2:%s/virtualenvtest' % - (sys.prefix)) + mock_get_python_lib.return_value = f'{sys.prefix}/virtualenvtest' + + # Inherit python path from current process + # Mock the current process python path + with mock.patch.dict(os.environ, {'PYTHONPATH': ':/data/test1:/data/test2'}): + python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', + inherit_from_parent=True, + inherit_parent_virtualenv=True) + + actual_path = python_path.strip(':').split(':') + self.assertEqual(len(actual_path), 7) + + # First entry should be lib/python3 dir from venv + self.assertEndsWith(actual_path[0], 'virtualenvs/dummy_pack/lib/python3.6') + # Second entry should be python3 site-packages dir from venv + self.assertEndsWith(actual_path[1], 'virtualenvs/dummy_pack/lib/python3.6/site-packages') + # Third entry should be actions/lib dir from pack root directory + self.assertEndsWith(actual_path[2], 'packs/dummy_pack/actions/lib') + # The paths from get_sandbox_python_path + self.assertEqual(actual_path[3], '') + self.assertEqual(actual_path[4], '/data/test1') + self.assertEqual(actual_path[5], '/data/test2') + # And the parent virtualenv + self.assertEqual(actual_path[6], f'{sys.prefix}/virtualenvtest')