From 087b067a3358b47c53ec807bed254f28e5ac7aa4 Mon Sep 17 00:00:00 2001 From: Amanda McGuinness Date: Thu, 3 Dec 2020 10:51:56 +0000 Subject: [PATCH 01/10] Initial changes --- conf/st2.conf.sample | 4 - contrib/packs/actions/download.yaml | 5 - contrib/packs/actions/install.meta.yaml | 5 - contrib/packs/actions/pack_mgmt/download.py | 5 +- .../actions/pack_mgmt/setup_virtualenv.py | 4 +- contrib/packs/actions/setup_virtualenv.yaml | 5 - contrib/packs/actions/workflows/install.yaml | 3 - contrib/packs/tests/test_action_download.py | 488 +----------------- st2api/st2api/controllers/v1/packs.py | 1 - st2client/st2client/commands/pack.py | 6 +- st2client/st2client/models/core.py | 3 +- st2common/st2common/cmd/download_pack.py | 6 +- st2common/st2common/cmd/install_pack.py | 9 +- .../st2common/cmd/setup_pack_virtualenv.py | 5 +- st2common/st2common/config.py | 9 - .../st2common/constants/error_messages.py | 9 - st2common/st2common/constants/pack.py | 6 - st2common/st2common/openapi.yaml | 4 - st2common/st2common/openapi.yaml.j2 | 4 - st2common/st2common/util/pack_management.py | 31 +- st2common/st2common/util/sandboxing.py | 85 --- st2common/st2common/util/virtualenvs.py | 35 +- st2common/tests/unit/test_util_sandboxing.py | 128 ----- st2common/tests/unit/test_virtualenvs.py | 43 -- .../st2reactor/container/process_container.py | 10 - tools/config_gen.py | 1 - 26 files changed, 25 insertions(+), 889 deletions(-) diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index 6e0abb00ca..a8111f59af 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -14,10 +14,6 @@ actions_pool_size = 60 logging = /etc/st2/logging.actionrunner.conf # List of pip options to be passed to "pip install" command when installing pack dependencies into pack virtual environment. pip_opts = # comma separated list allowed here. -# Python 3 binary which will be used by Python actions for packs which use Python 3 virtual environment. -python3_binary = /usr/bin/python3 -# Prefix for Python 3 installation (e.g. /opt/python3.6). If not specified, it tries to find Python 3 libraries in /usr/lib and /usr/local/lib. -python3_prefix = None # Python binary which will be used by Python actions. python_binary = /usr/bin/python # Default log level to use for Python runner actions. Can be overriden on invocation basis using "log_level" runner parameter. diff --git a/contrib/packs/actions/download.yaml b/contrib/packs/actions/download.yaml index 575d1ff45a..0849616b65 100644 --- a/contrib/packs/actions/download.yaml +++ b/contrib/packs/actions/download.yaml @@ -22,11 +22,6 @@ description: "Set to True to force install the pack and skip StackStorm version compatibility check" required: false default: false - python3: - type: "boolean" - description: "True to use Python 3 binary for this pack." - required: false - default: false dependency_list: type: "array" description: "Dependency list that needs to be downloaded." diff --git a/contrib/packs/actions/install.meta.yaml b/contrib/packs/actions/install.meta.yaml index 5d85a6ca04..1b8d0d572a 100644 --- a/contrib/packs/actions/install.meta.yaml +++ b/contrib/packs/actions/install.meta.yaml @@ -27,11 +27,6 @@ description: "Set to True to force install the pack and skip StackStorm version compatibility check and also delete and ignore lock file if one exists." required: false default: false - python3: - type: "boolean" - description: "Use Python 3 binary when creating a virtualenv for this pack." - required: false - default: false skip_dependencies: type: "boolean" description: "Set to True to skip pack dependency installations." diff --git a/contrib/packs/actions/pack_mgmt/download.py b/contrib/packs/actions/pack_mgmt/download.py index afe1c4db20..b4d888630b 100644 --- a/contrib/packs/actions/pack_mgmt/download.py +++ b/contrib/packs/actions/pack_mgmt/download.py @@ -63,7 +63,7 @@ def __init__(self, config=None, action_service=None): if self.proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None): os.environ['no_proxy'] = self.no_proxy - def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False, + def run(self, packs, abs_repo_base, verifyssl=True, force=False, dependency_list=None): result = {} pack_url = None @@ -73,7 +73,7 @@ def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False, pack_result = download_pack(pack=pack_dependency, abs_repo_base=abs_repo_base, verify_ssl=verifyssl, force=force, proxy_config=self.proxy_config, force_permissions=True, - use_python3=python3, logger=self.logger) + logger=self.logger) pack_url, pack_ref, pack_result = pack_result result[pack_ref] = pack_result else: @@ -82,7 +82,6 @@ def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False, verify_ssl=verifyssl, force=force, proxy_config=self.proxy_config, force_permissions=True, - use_python3=python3, logger=self.logger) pack_url, pack_ref, pack_result = pack_result result[pack_ref] = pack_result diff --git a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py index d60b8bbfe7..23f8a75ef7 100644 --- a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py +++ b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py @@ -74,7 +74,7 @@ def __init__(self, config=None, action_service=None): if self.proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None): os.environ['no_proxy'] = self.no_proxy - def run(self, packs, update=False, python3=False, no_download=True): + def run(self, packs, update=False, no_download=True): """ :param packs: A list of packs to create the environment for. :type: packs: ``list`` @@ -85,7 +85,7 @@ def run(self, packs, update=False, python3=False, no_download=True): for pack_name in packs: setup_pack_virtualenv(pack_name=pack_name, update=update, logger=self.logger, - proxy_config=self.proxy_config, use_python3=python3, + proxy_config=self.proxy_config, no_download=no_download) message = ('Successfully set up virtualenv for the following packs: %s' % diff --git a/contrib/packs/actions/setup_virtualenv.yaml b/contrib/packs/actions/setup_virtualenv.yaml index 6625cc70c5..18d1b3df15 100644 --- a/contrib/packs/actions/setup_virtualenv.yaml +++ b/contrib/packs/actions/setup_virtualenv.yaml @@ -14,11 +14,6 @@ type: "boolean" default: false description: "Check this option if the virtual environment already exists and if you only want to perform an update and installation of new dependencies. If you don't check this option, the virtual environment will be destroyed then re-created. If you check this and the virtual environment doesn't exist, it will create it." - python3: - type: "boolean" - description: "Use Python 3 binary when creating a virtualenv for this pack." - required: false - default: false no_download: type: "boolean" description: "Use version of pip which is already on the system instead of downloading the latest version from PyPi when creating a virtual environment." diff --git a/contrib/packs/actions/workflows/install.yaml b/contrib/packs/actions/workflows/install.yaml index 3d78668832..a67a754146 100644 --- a/contrib/packs/actions/workflows/install.yaml +++ b/contrib/packs/actions/workflows/install.yaml @@ -7,7 +7,6 @@ input: - register - env - force - - python3 - skip_dependencies - timeout @@ -30,7 +29,6 @@ tasks: input: packs: <% ctx().packs %> force: <% ctx().force %> - python3: <% ctx().python3 %> dependency_list: <% ctx().dependency_list %> next: - when: <% succeeded() %> @@ -98,7 +96,6 @@ tasks: input: packs: <% ctx().packs_list %> env: <% ctx().env %> - python3: <% ctx().python3 %> timeout: <% ctx().timeout %> next: - when: <% succeeded() %> diff --git a/contrib/packs/tests/test_action_download.py b/contrib/packs/tests/test_action_download.py index a5d84929fb..02d940bd7f 100644 --- a/contrib/packs/tests/test_action_download.py +++ b/contrib/packs/tests/test_action_download.py @@ -1,490 +1,4 @@ -#!/usr/bin/env python - -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import mock -import shutil -import tempfile -import hashlib - -from st2common.util.monkey_patch import use_select_poll_workaround -use_select_poll_workaround() - -from lockfile import LockFile -from lockfile import LockTimeout -from git.repo import Repo -from gitdb.exc import BadName -from st2common.services import packs as pack_service -from st2tests.base import BaseActionTestCase - -import st2common.util.pack_management -from st2common.util.pack_management import eval_repo_url - -from pack_mgmt.download import DownloadGitRepoAction - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) - -PACK_INDEX = { - "test": { - "version": "0.4.0", - "name": "test", - "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test", - "author": "st2-dev", - "keywords": ["some", "search", "another", "terms"], - "email": "info@stackstorm.com", - "description": "st2 pack to test package management pipeline" - }, - "test2": { - "version": "0.5.0", - "name": "test2", - "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test2", - "author": "stanley", - "keywords": ["some", "special", "terms"], - "email": "info@stackstorm.com", - "description": "another st2 pack to test package management pipeline" - }, - "test3": { - "version": "0.5.0", - "stackstorm_version": ">=1.6.0, <2.2.0", - "name": "test3", - "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test3", - "author": "stanley", - "keywords": ["some", "special", "terms"], - "email": "info@stackstorm.com", - "description": "another st2 pack to test package management pipeline" - }, - "test4": { - "version": "0.5.0", - "name": "test4", - "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test4", - "author": "stanley", - "keywords": ["some", "special", "terms"], "email": "info@stackstorm.com", - "description": "another st2 pack to test package management pipeline" - } -} - - -original_is_dir_func = os.path.isdir - - -def mock_is_dir_func(path): - """ - Mock function which returns True if path ends with .git - """ - if path.endswith('.git'): - return True - return original_is_dir_func(path) - - -def mock_get_gitref(repo, ref): - """ - Mock get_gitref function which return mocked object if ref passed is - PACK_INDEX['test']['version'] - """ - if PACK_INDEX['test']['version'] in ref: - if ref[0] == 'v': - return mock.MagicMock(hexsha=PACK_INDEX['test']['version']) - else: - return None - elif ref: - return mock.MagicMock(hexsha="abcDef") - else: - return None - - -@mock.patch.object(pack_service, 'fetch_pack_index', mock.MagicMock(return_value=(PACK_INDEX, {}))) -class DownloadGitRepoActionTestCase(BaseActionTestCase): - action_cls = DownloadGitRepoAction - - def setUp(self): - super(DownloadGitRepoActionTestCase, self).setUp() - - clone_from = mock.patch.object(Repo, 'clone_from') - - self.addCleanup(clone_from.stop) - self.clone_from = clone_from.start() - - self.expand_user_path = tempfile.mkdtemp() - expand_user = mock.patch.object(os.path, 'expanduser', - mock.MagicMock(return_value=self.expand_user_path)) - - self.addCleanup(expand_user.stop) - self.expand_user = expand_user.start() - - self.repo_base = tempfile.mkdtemp() - - self.repo_instance = mock.MagicMock() - type(self.repo_instance).active_branch = mock.Mock() - - def side_effect(url, to_path, **kwargs): - # Since we have no way to pass pack name here, we would have to derive it from repo url - fixture_name = url.split('/')[-1] - fixture_path = os.path.join(self._get_base_pack_path(), 'tests/fixtures', fixture_name) - shutil.copytree(fixture_path, to_path) - return self.repo_instance - - self.clone_from.side_effect = side_effect - - def tearDown(self): - shutil.rmtree(self.repo_base) - shutil.rmtree(self.expand_user()) - - def test_run_pack_download(self): - action = self.get_action_instance() - result = action.run(packs=['test'], abs_repo_base=self.repo_base) - temp_dir = hashlib.md5(PACK_INDEX['test']['repo_url'].encode()).hexdigest() - - self.assertEqual(result, {'test': 'Success.'}) - self.clone_from.assert_called_once_with(PACK_INDEX['test']['repo_url'], - os.path.join(os.path.expanduser('~'), temp_dir)) - self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test/pack.yaml'))) - - self.repo_instance.git.checkout.assert_called() - self.repo_instance.git.branch.assert_called() - self.repo_instance.git.checkout.assert_called() - - def test_run_pack_download_dependencies(self): - action = self.get_action_instance() - result = action.run(packs=['test'], dependency_list=['test2', 'test4'], - abs_repo_base=self.repo_base) - temp_dirs = [ - hashlib.md5(PACK_INDEX['test2']['repo_url'].encode()).hexdigest(), - hashlib.md5(PACK_INDEX['test4']['repo_url'].encode()).hexdigest() - ] - - self.assertEqual(result, {'test2': 'Success.', 'test4': 'Success.'}) - self.clone_from.assert_any_call(PACK_INDEX['test2']['repo_url'], - os.path.join(os.path.expanduser('~'), temp_dirs[0])) - self.clone_from.assert_any_call(PACK_INDEX['test4']['repo_url'], - os.path.join(os.path.expanduser('~'), temp_dirs[1])) - self.assertEqual(self.clone_from.call_count, 2) - self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test2/pack.yaml'))) - self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test4/pack.yaml'))) - - def test_run_pack_download_existing_pack(self): - action = self.get_action_instance() - action.run(packs=['test'], abs_repo_base=self.repo_base) - self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test/pack.yaml'))) - - result = action.run(packs=['test'], abs_repo_base=self.repo_base) - - self.assertEqual(result, {'test': 'Success.'}) - - def test_run_pack_download_multiple_packs(self): - action = self.get_action_instance() - result = action.run(packs=['test', 'test2'], abs_repo_base=self.repo_base) - temp_dirs = [ - hashlib.md5(PACK_INDEX['test']['repo_url'].encode()).hexdigest(), - hashlib.md5(PACK_INDEX['test2']['repo_url'].encode()).hexdigest() - ] - - self.assertEqual(result, {'test': 'Success.', 'test2': 'Success.'}) - self.clone_from.assert_any_call(PACK_INDEX['test']['repo_url'], - os.path.join(os.path.expanduser('~'), temp_dirs[0])) - self.clone_from.assert_any_call(PACK_INDEX['test2']['repo_url'], - os.path.join(os.path.expanduser('~'), temp_dirs[1])) - self.assertEqual(self.clone_from.call_count, 2) - self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test/pack.yaml'))) - self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test2/pack.yaml'))) - - @mock.patch.object(Repo, 'clone_from') - def test_run_pack_download_error(self, clone_from): - clone_from.side_effect = Exception('Something went terribly wrong during the clone') - - action = self.get_action_instance() - self.assertRaises(Exception, action.run, packs=['test'], abs_repo_base=self.repo_base) - - def test_run_pack_download_no_tag(self): - self.repo_instance.commit.side_effect = BadName - - action = self.get_action_instance() - self.assertRaises(ValueError, action.run, packs=['test=1.2.3'], - abs_repo_base=self.repo_base) - - def test_run_pack_lock_is_already_acquired(self): - action = self.get_action_instance() - temp_dir = hashlib.md5(PACK_INDEX['test']['repo_url'].encode()).hexdigest() - - original_acquire = LockFile.acquire - - def mock_acquire(self, timeout=None): - original_acquire(self, timeout=0.1) - - LockFile.acquire = mock_acquire - - try: - lock_file = LockFile('/tmp/%s' % (temp_dir)) - - # Acquire a lock (file) so acquire inside download will fail - with open(lock_file.lock_file, 'w') as fp: - fp.write('') - - expected_msg = 'Timeout waiting to acquire lock for' - self.assertRaisesRegexp(LockTimeout, expected_msg, action.run, packs=['test'], - abs_repo_base=self.repo_base) - finally: - os.unlink(lock_file.lock_file) - LockFile.acquire = original_acquire - - def test_run_pack_lock_is_already_acquired_force_flag(self): - # Lock is already acquired but force is true so it should be deleted and released - action = self.get_action_instance() - temp_dir = hashlib.md5(PACK_INDEX['test']['repo_url'].encode()).hexdigest() - - original_acquire = LockFile.acquire - - def mock_acquire(self, timeout=None): - original_acquire(self, timeout=0.1) - - LockFile.acquire = mock_acquire - - try: - lock_file = LockFile('/tmp/%s' % (temp_dir)) - - # Acquire a lock (file) so acquire inside download will fail - with open(lock_file.lock_file, 'w') as fp: - fp.write('') - - result = action.run(packs=['test'], abs_repo_base=self.repo_base, force=True) - finally: - LockFile.acquire = original_acquire - - self.assertEqual(result, {'test': 'Success.'}) - - def test_run_pack_download_v_tag(self): - def side_effect(ref): - if ref[0] != 'v': - raise BadName() - return mock.MagicMock(hexsha='abcdef') - - self.repo_instance.commit.side_effect = side_effect - self.repo_instance.git = mock.MagicMock( - branch=(lambda *args: 'master'), - checkout=(lambda *args: True) - ) - - action = self.get_action_instance() - result = action.run(packs=['test=1.2.3'], abs_repo_base=self.repo_base) - - self.assertEqual(result, {'test': 'Success.'}) - - @mock.patch.object(st2common.util.pack_management, 'get_valid_versions_for_repo', - mock.Mock(return_value=['1.0.0', '2.0.0'])) - def test_run_pack_download_invalid_version(self): - self.repo_instance.commit.side_effect = lambda ref: None - - action = self.get_action_instance() - - expected_msg = ('is not a valid version, hash, tag or branch.*?' - 'Available versions are: 1.0.0, 2.0.0.') - self.assertRaisesRegexp(ValueError, expected_msg, action.run, - packs=['test=2.2.3'], abs_repo_base=self.repo_base) - - def test_download_pack_stackstorm_version_identifier_check(self): - action = self.get_action_instance() - - # Version is satisfied - st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '2.0.0' - - result = action.run(packs=['test3'], abs_repo_base=self.repo_base) - self.assertEqual(result['test3'], 'Success.') - - # Pack requires a version which is not satisfied by current StackStorm version - st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '2.2.0' - expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' - 'current version is "2.2.0"') - self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], - abs_repo_base=self.repo_base) - - st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '2.3.0' - expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' - 'current version is "2.3.0"') - self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], - abs_repo_base=self.repo_base) - - st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '1.5.9' - expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' - 'current version is "1.5.9"') - self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], - abs_repo_base=self.repo_base) - - st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '1.5.0' - expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' - 'current version is "1.5.0"') - self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], - abs_repo_base=self.repo_base) - - # Version is not met, but force=true parameter is provided - st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '1.5.0' - result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=True) - self.assertEqual(result['test3'], 'Success.') - - def test_download_pack_python_version_check(self): - action = self.get_action_instance() - - # No python_versions attribute specified in the metadata file - with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ - mock_get_pack_metadata: - mock_get_pack_metadata.return_value = { - 'name': 'test3', - 'stackstorm_version': '', - 'python_versions': [] - } - - st2common.util.pack_management.six.PY2 = True - st2common.util.pack_management.six.PY3 = False - st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.11' - - result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False) - self.assertEqual(result['test3'], 'Success.') - - # Pack works with Python 2.x installation is running 2.7 - with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ - mock_get_pack_metadata: - mock_get_pack_metadata.return_value = { - 'name': 'test3', - 'stackstorm_version': '', - 'python_versions': ['2'] - } - - st2common.util.pack_management.six.PY2 = True - st2common.util.pack_management.six.PY3 = False - st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5' - - result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False) - self.assertEqual(result['test3'], 'Success.') - - st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.12' - - result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False) - self.assertEqual(result['test3'], 'Success.') - - # Pack works with Python 2.x installation is running 3.5 - with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ - mock_get_pack_metadata: - mock_get_pack_metadata.return_value = { - 'name': 'test3', - 'stackstorm_version': '', - 'python_versions': ['2'] - } - - st2common.util.pack_management.six.PY2 = False - st2common.util.pack_management.six.PY3 = True - - st2common.util.pack_management.CURRENT_PYTHON_VERSION = '3.5.2' - - expected_msg = (r'Pack "test3" requires Python 2.x, but current Python version is ' - '"3.5.2"') - self.assertRaisesRegexp(ValueError, expected_msg, action.run, - packs=['test3'], abs_repo_base=self.repo_base, force=False) - - # Pack works with Python 3.x installation is running 2.7 - with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ - mock_get_pack_metadata: - mock_get_pack_metadata.return_value = { - 'name': 'test3', - 'stackstorm_version': '', - 'python_versions': ['3'] - } - - st2common.util.pack_management.six.PY2 = True - st2common.util.pack_management.six.PY3 = False - st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.2' - - expected_msg = (r'Pack "test3" requires Python 3.x, but current Python version is ' - '"2.7.2"') - self.assertRaisesRegexp(ValueError, expected_msg, action.run, - packs=['test3'], abs_repo_base=self.repo_base, force=False) - - # Pack works with Python 2.x and 3.x installation is running 2.7 and 3.6.1 - with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ - mock_get_pack_metadata: - mock_get_pack_metadata.return_value = { - 'name': 'test3', - 'stackstorm_version': '', - 'python_versions': ['2', '3'] - } - - st2common.util.pack_management.six.PY2 = True - st2common.util.pack_management.six.PY3 = False - st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5' - - result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False) - self.assertEqual(result['test3'], 'Success.') - - st2common.util.pack_management.six.PY2 = False - st2common.util.pack_management.six.PY3 = True - st2common.util.pack_management.CURRENT_PYTHON_VERSION = '3.6.1' - - result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False) - self.assertEqual(result['test3'], 'Success.') - - # StackStorm is running under Python 2, Pack requires Python 3 and --python3 flag is used - with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ - mock_get_pack_metadata: - mock_get_pack_metadata.return_value = { - 'name': 'test3', - 'stackstorm_version': '', - 'python_versions': ['3'] - } - - st2common.util.pack_management.six.PY2 = True - st2common.util.pack_management.six.PY3 = False - st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5' - - result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False, - python3=True) - self.assertEqual(result['test3'], 'Success.') - - with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ - mock_get_pack_metadata: - mock_get_pack_metadata.return_value = { - 'name': 'test3', - 'stackstorm_version': '', - 'python_versions': ['2', '3'] - } - - st2common.util.pack_management.six.PY2 = True - st2common.util.pack_management.six.PY3 = False - st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5' - - result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False, - python3=True) - self.assertEqual(result['test3'], 'Success.') - - # StackStorm is running under Python 2, pack requires Python 3 and --python3 flag is used - with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ - mock_get_pack_metadata: - mock_get_pack_metadata.return_value = { - 'name': 'test3', - 'stackstorm_version': '', - 'python_versions': ['2'] - } - - st2common.util.pack_management.six.PY2 = True - st2common.util.pack_management.six.PY3 = False - st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5' - - expected_msg = (r'Pack "test3" requires Python 2.x, but --python3 flag is used') - self.assertRaisesRegexp(ValueError, expected_msg, action.run, - packs=['test3'], abs_repo_base=self.repo_base, force=False, - python3=True) - - def test_resolve_urls(self): +def test_resolve_urls(self): url = eval_repo_url( "https://github.com/StackStorm-Exchange/stackstorm-test") self.assertEqual(url, "https://github.com/StackStorm-Exchange/stackstorm-test") diff --git a/st2api/st2api/controllers/v1/packs.py b/st2api/st2api/controllers/v1/packs.py index 129ccd5cda..6193a3f01f 100644 --- a/st2api/st2api/controllers/v1/packs.py +++ b/st2api/st2api/controllers/v1/packs.py @@ -98,7 +98,6 @@ class PackInstallController(ActionExecutionsControllerMixin): def post(self, pack_install_request, requester_user=None): parameters = { 'packs': pack_install_request.packs, - 'python3': pack_install_request.python3 } if pack_install_request.force: diff --git a/st2client/st2client/commands/pack.py b/st2client/st2client/commands/pack.py index 310795ae6b..617148642f 100644 --- a/st2client/st2client/commands/pack.py +++ b/st2client/st2client/commands/pack.py @@ -189,10 +189,6 @@ def __init__(self, resource, *args, **kwargs): metavar='pack', help='Name of the %s in Exchange, or a git repo URL.' % resource.get_plural_display_name().lower()) - self.parser.add_argument('--python3', - action='store_true', - default=False, - help='Use Python 3 binary for pack virtual environment.') self.parser.add_argument('--force', action='store_true', default=False, @@ -210,7 +206,7 @@ def run(self, args, **kwargs): if not is_structured_output: self._get_content_counts_for_pack(args, **kwargs) - return self.manager.install(args.packs, python3=args.python3, force=args.force, + return self.manager.install(args.packs, force=args.force, skip_dependencies=args.skip_dependencies, **kwargs) def _get_content_counts_for_pack(self, args, **kwargs): diff --git a/st2client/st2client/models/core.py b/st2client/st2client/models/core.py index e6fec6da70..d102d3158f 100644 --- a/st2client/st2client/models/core.py +++ b/st2client/st2client/models/core.py @@ -507,12 +507,11 @@ class AsyncRequest(Resource): class PackResourceManager(ResourceManager): @add_auth_token_to_kwargs_from_env - def install(self, packs, force=False, python3=False, skip_dependencies=False, **kwargs): + def install(self, packs, force=False, skip_dependencies=False, **kwargs): url = '/%s/install' % (self.resource.get_url_path_name()) payload = { 'packs': packs, 'force': force, - 'python3': python3, 'skip_dependencies': skip_dependencies } response = self.client.post(url, payload, **kwargs) diff --git a/st2common/st2common/cmd/download_pack.py b/st2common/st2common/cmd/download_pack.py index e24432ff89..b22a3f0467 100644 --- a/st2common/st2common/cmd/download_pack.py +++ b/st2common/st2common/cmd/download_pack.py @@ -41,8 +41,6 @@ def _register_cli_opts(): cfg.BoolOpt('force', default=False, help='True to force pack download and ignore download ' 'lock file if it exists.'), - cfg.BoolOpt('use-python3', default=False, - help='True to use Python3 binary.') ] do_register_cli_opts(cli_opts) @@ -57,15 +55,13 @@ def main(argv): packs = cfg.CONF.pack verify_ssl = cfg.CONF.verify_ssl force = cfg.CONF.force - use_python3 = cfg.CONF.use_python3 proxy_config = get_and_set_proxy_config() for pack in packs: LOG.info('Installing pack "%s"' % (pack)) result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, - proxy_config=proxy_config, force_permissions=True, - use_python3=use_python3) + proxy_config=proxy_config, force_permissions=True) # Raw pack name excluding the version pack_name = result[1] diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py index bef02152bd..861d0d4041 100644 --- a/st2common/st2common/cmd/install_pack.py +++ b/st2common/st2common/cmd/install_pack.py @@ -42,9 +42,6 @@ def _register_cli_opts(): cfg.BoolOpt('force', default=False, help='True to force pack installation and ignore install ' 'lock file if it exists.'), - cfg.BoolOpt('use-python3', default=False, - help='True to use Python3 binary when creating virtualenv ' - 'for this pack.'), ] do_register_cli_opts(cli_opts) @@ -59,7 +56,6 @@ def main(argv): packs = cfg.CONF.pack verify_ssl = cfg.CONF.verify_ssl force = cfg.CONF.force - use_python3 = cfg.CONF.use_python3 proxy_config = get_and_set_proxy_config() @@ -67,8 +63,7 @@ def main(argv): # 1. Download the pack LOG.info('Installing pack "%s"' % (pack)) result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, - proxy_config=proxy_config, force_permissions=True, - use_python3=use_python3) + proxy_config=proxy_config, force_permissions=True) # Raw pack name excluding the version pack_name = result[1] @@ -84,7 +79,7 @@ def main(argv): # 2. Setup pack virtual environment LOG.info('Setting up virtualenv for pack "%s"' % (pack_name)) setup_pack_virtualenv(pack_name=pack_name, update=False, logger=LOG, - proxy_config=proxy_config, use_python3=use_python3, + proxy_config=proxy_config, no_download=True) LOG.info('Successfully set up virtualenv for pack "%s"' % (pack_name)) diff --git a/st2common/st2common/cmd/setup_pack_virtualenv.py b/st2common/st2common/cmd/setup_pack_virtualenv.py index d82e924d5c..626bb389af 100644 --- a/st2common/st2common/cmd/setup_pack_virtualenv.py +++ b/st2common/st2common/cmd/setup_pack_virtualenv.py @@ -39,8 +39,6 @@ def _register_cli_opts(): 'you don\'t check this option, the virtual environment will be destroyed ' 'then re-created. If you check this and the virtual environment doesn\'t ' 'exist, it will create it..')), - cfg.BoolOpt('python3', default=False, - help='Use Python 3 binary when creating a virtualenv for this pack.'), ] do_register_cli_opts(cli_opts) @@ -54,7 +52,6 @@ def main(argv): packs = cfg.CONF.pack update = cfg.CONF.update - use_python3 = cfg.CONF.python3 proxy_config = get_and_set_proxy_config() @@ -62,7 +59,7 @@ def main(argv): # Setup pack virtual environment LOG.info('Setting up virtualenv for pack "%s"' % (pack)) setup_pack_virtualenv(pack_name=pack, update=update, logger=LOG, - proxy_config=proxy_config, use_python3=use_python3, + proxy_config=proxy_config, no_download=True) LOG.info('Successfully set up virtualenv for pack "%s"' % (pack)) diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index d02ded345d..c0f5148f1c 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -345,7 +345,6 @@ def register_opts(ignore_errors=False): # Runner options default_python_bin_path = sys.executable - default_python3_bin_path = find_executable('python3') base_dir = os.path.dirname(os.path.realpath(default_python_bin_path)) default_virtualenv_bin_path = os.path.join(base_dir, 'virtualenv') @@ -359,14 +358,6 @@ def register_opts(ignore_errors=False): cfg.StrOpt( 'python_binary', default=default_python_bin_path, help='Python binary which will be used by Python actions.'), - cfg.StrOpt( - 'python3_binary', default=default_python3_bin_path, - help='Python 3 binary which will be used by Python actions for packs which ' - 'use Python 3 virtual environment.'), - cfg.StrOpt( - 'python3_prefix', default=None, - help='Prefix for Python 3 installation (e.g. /opt/python3.6). If not specified, it ' - 'tries to find Python 3 libraries in /usr/lib and /usr/local/lib.'), cfg.StrOpt( 'virtualenv_binary', default=default_virtualenv_bin_path, help='Virtualenv binary which should be used to create pack virtualenvs.'), diff --git a/st2common/st2common/constants/error_messages.py b/st2common/st2common/constants/error_messages.py index 749f7df659..7aa56c4025 100644 --- a/st2common/st2common/constants/error_messages.py +++ b/st2common/st2common/constants/error_messages.py @@ -15,7 +15,6 @@ __all__ = [ 'PACK_VIRTUALENV_DOESNT_EXIST', - 'PACK_VIRTUALENV_USES_PYTHON3', 'PYTHON2_DEPRECATION' ] @@ -26,14 +25,6 @@ "st2 run packs.setup_virtualenv packs=%(pack)s" ''' -PACK_VIRTUALENV_USES_PYTHON3 = ''' -Virtual environment (%(virtualenv_path)s) for pack "%(pack)s" is using Python 3. -Using Python 3 virtual environments in mixed deployments is only supported for Python runner -actions and not sensors. If you want to run this sensor, please re-recreate the -virtual environment with python2 binary: -"st2 run packs.setup_virtualenv packs=%(pack)s python3=false" -''' - PYTHON2_DEPRECATION = 'DEPRECATION WARNING. Support for python 2 will be removed in future ' \ 'StackStorm releases. Please ensure that all packs used are python ' \ '3 compatible. Your StackStorm installation may be upgraded from ' \ diff --git a/st2common/st2common/constants/pack.py b/st2common/st2common/constants/pack.py index f1e11d42d4..91ae5a5e2c 100644 --- a/st2common/st2common/constants/pack.py +++ b/st2common/st2common/constants/pack.py @@ -87,12 +87,6 @@ 'six>=1.9.0,<2.0' ] -# Python requirements which are common to all the packs and need to be installed -# for Python 3 pack virtual environments to work -BASE_PACK_PYTHON3_REQUIREMENTS = [ - 'pyyaml>=5.1,<5.2' -] - # Name of the pack manifest file MANIFEST_FILE_NAME = 'pack.yaml' diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index 3d6a1ec1f1..f1a116c3d6 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -5068,10 +5068,6 @@ definitions: type: array items: type: string - python3: - description: Use Python 3 binary for pack virtual environment. - type: boolean - default: false force: description: Force pack installation. type: boolean diff --git a/st2common/st2common/openapi.yaml.j2 b/st2common/st2common/openapi.yaml.j2 index 1d88c0018a..498e0e54a2 100644 --- a/st2common/st2common/openapi.yaml.j2 +++ b/st2common/st2common/openapi.yaml.j2 @@ -5064,10 +5064,6 @@ definitions: type: array items: type: string - python3: - description: Use Python 3 binary for pack virtual environment. - type: boolean - default: false force: description: Force pack installation. type: boolean diff --git a/st2common/st2common/util/pack_management.py b/st2common/st2common/util/pack_management.py index f435655e5c..d12b9a5f6c 100644 --- a/st2common/st2common/util/pack_management.py +++ b/st2common/st2common/util/pack_management.py @@ -71,7 +71,7 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, force=False, proxy_config=None, force_owner_group=True, force_permissions=True, - use_python3=False, logger=LOG): + logger=LOG): """ Download the pack and move it to /opt/stackstorm/packs. @@ -91,21 +91,11 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, :param force: Force the installation and ignore / delete the lock file if it already exists. :type force: ``bool`` - :param use_python3: True if a python3 binary should be used for this pack. - :type use_python3: ``bool`` - :return: (pack_url, pack_ref, result) :rtype: tuple """ proxy_config = proxy_config or {} - # Python3 binary check - python3_binary = cfg.CONF.actionrunner.python3_binary - if use_python3 and not python3_binary: - msg = ('Requested to use Python 3, but python3 binary not found on the system or ' - 'actionrunner.python3 config option is not configured correctly.') - raise ValueError(msg) - try: pack_url, pack_version = get_repo_url(pack, proxy_config=proxy_config) except Exception as e: @@ -162,7 +152,7 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, # 2. Verify that the pack version if compatible with current StackStorm version if not force: - verify_pack_version(pack_metadata=pack_metadata, use_python3=use_python3) + verify_pack_version(pack_metadata=pack_metadata) # 3. Move pack to the final location move_result = move_pack(abs_repo_base=abs_repo_base, pack_name=pack_ref, @@ -428,7 +418,7 @@ def is_desired_pack(abs_pack_path, pack_name): return (True, '') -def verify_pack_version(pack_metadata, use_python3=False): +def verify_pack_version(pack_metadata): """ Verify that the pack works with the currently running StackStorm version. """ @@ -447,17 +437,12 @@ def verify_pack_version(pack_metadata, use_python3=False): raise ValueError(msg) if supported_python_versions: - if set(supported_python_versions) == set(['2']) and (not six.PY2 or use_python3): - if use_python3: - msg = ('Pack "%s" requires Python 2.x, but --python3 flag is used. ' - 'You can override this restriction by providing the "force" flag, but ' - 'the pack is not guaranteed to work.' % (pack_name)) - else: - msg = ('Pack "%s" requires Python 2.x, but current Python version is "%s". ' - 'You can override this restriction by providing the "force" flag, but ' - 'the pack is not guaranteed to work.' % (pack_name, CURRENT_PYTHON_VERSION)) + if set(supported_python_versions) == set(['2']) and (not six.PY2): + msg = ('Pack "%s" requires Python 2.x, but current Python version is "%s". ' + 'You can override this restriction by providing the "force" flag, but ' + 'the pack is not guaranteed to work.' % (pack_name, CURRENT_PYTHON_VERSION)) raise ValueError(msg) - elif set(supported_python_versions) == set(['3']) and (not six.PY3 and not use_python3): + elif set(supported_python_versions) == set(['3']) and (not six.PY3): msg = ('Pack "%s" requires Python 3.x, but current Python version is "%s". ' 'You can override this restriction by providing the "force" flag, but ' 'the pack is not guaranteed to work.' % (pack_name, CURRENT_PYTHON_VERSION)) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 930197b781..7691cdc9e4 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -36,8 +36,6 @@ 'get_sandbox_python_path_for_python_action', 'get_sandbox_path', 'get_sandbox_virtualenv_path', - - 'is_pack_virtualenv_using_python3' ] @@ -143,92 +141,9 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, pack_base_path = get_pack_base_path(pack_name=pack) virtualenv_path = get_sandbox_virtualenv_path(pack=pack) - if not virtualenv_path: - return sandbox_python_path - - uses_python3, virtualenv_directories = is_pack_virtualenv_using_python3(pack=pack) - if uses_python3: - # Add Python 3 lib directory (lib/python3.x) in front of the PYTHONPATH. This way we avoid - # issues with scripts trying to use packages / modules from Python 2.7 site-packages - # directory instead of the versions from Python 3 stdlib. - pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/') - pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') - python3_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0]) - - # Add Python 3 site-packages directory (lib/python3.x/site-packages) in front of the Python - # 2.7 system site-packages This is important because we want Python 3 compatible libraries - # to be used from the pack virtual environment and not system ones. - python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path, - virtualenv_directories[0], - 'site-packages') - - # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv - # one (e.g. /usr/lib/python3.6) - # NOTE: We can't simply use sys.prefix dir since it will be set to /opt/stackstorm/st2 - - system_prefix_dirs = [] - # Take custom prefix into account (if specified) - if cfg.CONF.actionrunner.python3_prefix: - system_prefix_dirs.append(cfg.CONF.actionrunner.python3_prefix) - - # By default, Python libs are installed either in /usr/lib/python3.x or - # /usr/local/lib/python3.x - system_prefix_dirs.extend(['/usr/lib', '/usr/local/lib']) - - for system_prefix_dir in system_prefix_dirs: - python3_system_lib_directory = os.path.join(system_prefix_dir, - virtualenv_directories[0]) - - if os.path.exists(python3_system_lib_directory): - break - - if not python3_system_lib_directory or not os.path.exists(python3_system_lib_directory): - python3_system_lib_directory = None - - full_sandbox_python_path = [] - - # NOTE: Order here is very important for imports to function correctly - if python3_system_lib_directory: - full_sandbox_python_path.append(python3_system_lib_directory) - - full_sandbox_python_path.append(python3_lib_directory) - full_sandbox_python_path.append(python3_site_packages_directory) - full_sandbox_python_path.append(pack_actions_lib_paths) - full_sandbox_python_path.append(sandbox_python_path) - - sandbox_python_path = ':'.join(full_sandbox_python_path) - return sandbox_python_path -def is_pack_virtualenv_using_python3(pack): - """ - Return True if a particular pack virtual environment is using Python 3. - - :return: (uses_python3_bool, virtualenv_lib_directories) - :rtype: ``tuple`` - """ - # If python3.? directory exists in pack virtualenv lib/ path it means Python 3 is used by - # that virtual environment and we take that in to account when constructing PYTHONPATH - virtualenv_path = get_sandbox_virtualenv_path(pack=pack) - - if virtualenv_path and os.path.isdir(virtualenv_path): - pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') - - if not os.path.exists(pack_virtualenv_lib_path): - return False, None - - virtualenv_directories = os.listdir(pack_virtualenv_lib_path) - virtualenv_directories = [dir_name for dir_name in virtualenv_directories if - fnmatch.fnmatch(dir_name, 'python3*')] - uses_python3 = bool(virtualenv_directories) - else: - uses_python3 = False - virtualenv_directories = None - - return uses_python3, virtualenv_directories - - def get_sandbox_virtualenv_path(pack): """ Return a path to the virtual environment for the provided pack. diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py index 88bdc49a73..a8e504ba4c 100644 --- a/st2common/st2common/util/virtualenvs.py +++ b/st2common/st2common/util/virtualenvs.py @@ -29,7 +29,6 @@ from st2common import log as logging from st2common.constants.pack import PACK_REF_WHITELIST_REGEX from st2common.constants.pack import BASE_PACK_REQUIREMENTS -from st2common.constants.pack import BASE_PACK_PYTHON3_REQUIREMENTS from st2common.util.shell import run_command from st2common.util.shell import quote_unix from st2common.util.compat import to_ascii @@ -46,7 +45,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True, include_setuptools=True, include_wheel=True, proxy_config=None, - use_python3=False, no_download=True, force_owner_group=True): + no_download=True, force_owner_group=True): """ Setup virtual environment for the provided pack. @@ -60,9 +59,6 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True :param logger: Optional logger instance to use. If not provided it defaults to the module level logger. - :param use_python3: Use Python3 binary when creating virtualenv for this pack. - :type use_python3: ``bool`` - :param no_download: Do not download and install latest version of pre-installed packages such as pip and distutils. :type no_download: ``bool`` @@ -95,7 +91,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path)) create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip, include_setuptools=include_setuptools, include_wheel=include_wheel, - use_python3=use_python3, no_download=no_download) + no_download=no_download) # 2. Install base requirements which are common to all the packs logger.debug('Installing base requirements') @@ -103,14 +99,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True install_requirement(virtualenv_path=virtualenv_path, requirement=requirement, proxy_config=proxy_config, logger=logger) - # 3. Install base Python 3 requirements which are common to all the packs - if use_python3: - logger.debug('Installing base Python 3 requirements') - for requirement in BASE_PACK_PYTHON3_REQUIREMENTS: - install_requirement(virtualenv_path=virtualenv_path, requirement=requirement, - proxy_config=proxy_config, logger=logger) - - # 4. Install pack-specific requirements + # 3. Install pack-specific requirements requirements_file_path = os.path.join(pack_path, 'requirements.txt') has_requirements = os.path.isfile(requirements_file_path) @@ -124,7 +113,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True else: logger.debug('No pack specific requirements found') - # 5. Set the owner group + # 4. Set the owner group if force_owner_group: apply_pack_owner_group(pack_path=virtualenv_path) @@ -134,7 +123,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_setuptools=True, - include_wheel=True, use_python3=False, no_download=True): + include_wheel=True, no_download=True): """ :param include_pip: Include pip binary and package in the newely created virtual environment. :type include_pip: ``bool`` @@ -146,9 +135,6 @@ def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_se :param include_wheel: Include wheel in the newely created virtual environment. :type include_wheel : ``bool`` - :param use_python3: Use Python3 binary when creating virtualenv for this pack. - :type use_python3: ``bool`` - :param no_download: Do not download and install latest version of pre-installed packages such as pip and distutils. :type no_download: ``bool`` @@ -157,7 +143,6 @@ def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_se logger = logger or LOG python_binary = cfg.CONF.actionrunner.python_binary - python3_binary = cfg.CONF.actionrunner.python3_binary virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts or [] @@ -172,15 +157,7 @@ def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_se cmd = [virtualenv_binary] - if use_python3 and not python3_binary: - msg = ('Requested to use Python 3, but python3 binary not found on the system or ' - 'actionrunner.python3 config option is not configured correctly.') - raise ValueError(msg) - - if use_python3: - cmd.extend(['-p', python3_binary]) - else: - cmd.extend(['-p', python_binary]) + cmd.extend(['-p', python_binary]) cmd.extend(virtualenv_opts) diff --git a/st2common/tests/unit/test_util_sandboxing.py b/st2common/tests/unit/test_util_sandboxing.py index 52b1fb4a44..8a713740cb 100644 --- a/st2common/tests/unit/test_util_sandboxing.py +++ b/st2common/tests/unit/test_util_sandboxing.py @@ -26,7 +26,6 @@ from st2common.util.sandboxing import get_sandbox_python_path from st2common.util.sandboxing import get_sandbox_python_path_for_python_action from st2common.util.sandboxing import get_sandbox_python_binary_path -from st2common.util.sandboxing import is_pack_virtualenv_using_python3 import st2tests.config as tests_config @@ -106,130 +105,3 @@ def test_get_sandbox_python_path(self, mock_get_python_lib): inherit_parent_virtualenv=True) self.assertEqual(python_path, ':/data/test1:/data/test2:%s/virtualenvtest' % (sys.prefix)) - - @mock.patch('os.path.isdir', mock.Mock(return_value=True)) - @mock.patch('os.listdir', mock.Mock(return_value=['python2.7'])) - @mock.patch('st2common.util.sandboxing.get_python_lib') - def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self, - mock_get_python_lib): - self.assertFalse(is_pack_virtualenv_using_python3(pack='dummy_pack')[0]) - - # No inheritance - python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', - inherit_from_parent=False, - inherit_parent_virtualenv=False) - - self.assertEqual(python_path, ':') - - # Inherit python path from current process - # Mock the current process python path - os.environ['PYTHONPATH'] = ':/data/test1:/data/test2' - - python_path = get_sandbox_python_path(inherit_from_parent=True, - inherit_parent_virtualenv=False) - self.assertEqual(python_path, ':/data/test1:/data/test2') - - # Inherit from current process and from virtualenv (not running inside virtualenv) - del sys.real_prefix - - python_path = get_sandbox_python_path(inherit_from_parent=True, - inherit_parent_virtualenv=False) - self.assertEqual(python_path, ':/data/test1:/data/test2') - - # Inherit from current process and from virtualenv (running inside virtualenv) - sys.real_prefix = '/usr' - mock_get_python_lib.return_value = sys.prefix + '/virtualenvtest' - python_path = get_sandbox_python_path(inherit_from_parent=True, - inherit_parent_virtualenv=True) - self.assertEqual(python_path, ':/data/test1:/data/test2:%s/virtualenvtest' % - (sys.prefix)) - - @mock.patch('os.path.exists', mock.Mock(return_value=True)) - @mock.patch('os.path.isdir', mock.Mock(return_value=True)) - @mock.patch('os.listdir', mock.Mock(return_value=['python3.6'])) - @mock.patch('st2common.util.sandboxing.get_python_lib') - @mock.patch('st2common.util.sandboxing.get_pack_base_path', - mock.Mock(return_value='/tmp/packs/dummy_pack')) - @mock.patch('st2common.util.sandboxing.get_sandbox_virtualenv_path', - mock.Mock(return_value='/tmp/virtualenvs/dummy_pack')) - def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self, - mock_get_python_lib): - self.assertTrue(is_pack_virtualenv_using_python3(pack='dummy_pack')[0]) - - # No inheritance - python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', - inherit_from_parent=False, - inherit_parent_virtualenv=False) - - split = python_path.strip(':').split(':') - self.assertEqual(len(split), 4) - - # First entry should be system lib/python3 dir - self.assertIn('/usr/lib/python3.6', split[0]) - - # Second entry should be lib/python3 dir from venv - self.assertIn('virtualenvs/dummy_pack/lib/python3.6', split[1]) - - # Third entry should be python3 site-packages dir from venv - self.assertIn('virtualenvs/dummy_pack/lib/python3.6/site-packages', split[2]) - - # Fourth entry should be actions/lib dir from pack root directory - self.assertIn('packs/dummy_pack/actions/lib/', split[3]) - - # Inherit python path from current process - # Mock the current process python path - os.environ['PYTHONPATH'] = ':/data/test1:/data/test2' - - python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', - inherit_from_parent=True, - inherit_parent_virtualenv=False) - expected = ('/usr/lib/python3.6:' - '/tmp/virtualenvs/dummy_pack/lib/python3.6:' - '/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:' - '/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2') - self.assertEqual(python_path, expected) - - # Inherit from current process and from virtualenv (not running inside virtualenv) - del sys.real_prefix - - python_path = get_sandbox_python_path(inherit_from_parent=True, - inherit_parent_virtualenv=False) - self.assertEqual(python_path, ':/data/test1:/data/test2') - - # Inherit from current process and from virtualenv (running inside virtualenv) - sys.real_prefix = '/usr' - mock_get_python_lib.return_value = sys.prefix + '/virtualenvtest' - python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', - inherit_from_parent=True, - inherit_parent_virtualenv=True) - - expected = ('/usr/lib/python3.6:' - '/tmp/virtualenvs/dummy_pack/lib/python3.6:' - '/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:' - '/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2:' - '%s/virtualenvtest' % (sys.prefix)) - self.assertEqual(python_path, expected) - - # Custom prefix specified in the config - cfg.CONF.set_override(name='python3_prefix', override='/opt/lib', - group='actionrunner') - - # No inheritance - python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', - inherit_from_parent=False, - inherit_parent_virtualenv=False) - - split = python_path.strip(':').split(':') - self.assertEqual(len(split), 4) - - # First entry should be system lib/python3 dir - self.assertIn('/opt/lib/python3.6', split[0]) - - # Second entry should be lib/python3 dir from venv - self.assertIn('virtualenvs/dummy_pack/lib/python3.6', split[1]) - - # Third entry should be python3 site-packages dir from venv - self.assertIn('virtualenvs/dummy_pack/lib/python3.6/site-packages', split[2]) - - # Fourth entry should be actions/lib dir from pack root directory - self.assertIn('packs/dummy_pack/actions/lib/', split[3]) diff --git a/st2common/tests/unit/test_virtualenvs.py b/st2common/tests/unit/test_virtualenvs.py index b7d6f37a3e..90c0f4e989 100644 --- a/st2common/tests/unit/test_virtualenvs.py +++ b/st2common/tests/unit/test_virtualenvs.py @@ -293,49 +293,6 @@ def test_install_requirements_with_https_proxy_no_cert(self): } virtualenvs.run_command.assert_called_once_with(**expected_args) - @mock.patch.object(virtualenvs, 'run_command') - def test_setup_pack_virtualenv_use_python3_binary(self, mock_run_command): - mock_run_command.return_value = 0, '', '' - - cfg.CONF.set_override(name='python_binary', group='actionrunner', - override='/usr/bin/python2.7') - cfg.CONF.set_override(name='python3_binary', group='actionrunner', - override='/usr/bin/python3') - - pack_name = 'dummy_pack_2' - - # Python 2 - setup_pack_virtualenv(pack_name=pack_name, update=False, - include_setuptools=False, include_wheel=False, - use_python3=False) - - actual_cmd = mock_run_command.call_args_list[0][1]['cmd'] - actual_cmd = ' '.join(actual_cmd) - - self.assertEqual(mock_run_command.call_count, 2) - self.assertIn('-p /usr/bin/python2.7', actual_cmd) - - mock_run_command.reset_mock() - - # Python 3 - setup_pack_virtualenv(pack_name=pack_name, update=False, - include_setuptools=False, include_wheel=False, - use_python3=True) - - self.assertEqual(mock_run_command.call_count, 3) - - actual_cmd = mock_run_command.call_args_list[0][1]['cmd'] - actual_cmd = ' '.join(actual_cmd) - self.assertIn('-p /usr/bin/python3', actual_cmd) - - actual_cmd = mock_run_command.call_args_list[1][1]['cmd'] - actual_cmd = ' '.join(actual_cmd) - self.assertIn('pip install pyyaml', actual_cmd) - - actual_cmd = mock_run_command.call_args_list[2][1]['cmd'] - actual_cmd = ' '.join(actual_cmd) - self.assertIn('pip install', actual_cmd) - def assertVirtualenvExists(self, virtualenv_dir): self.assertTrue(os.path.exists(virtualenv_dir)) self.assertTrue(os.path.isdir(virtualenv_dir)) diff --git a/st2reactor/st2reactor/container/process_container.py b/st2reactor/st2reactor/container/process_container.py index 34b9535063..f8f1638d71 100644 --- a/st2reactor/st2reactor/container/process_container.py +++ b/st2reactor/st2reactor/container/process_container.py @@ -29,7 +29,6 @@ from st2common import log as logging from st2common.util import concurrency from st2common.constants.error_messages import PACK_VIRTUALENV_DOESNT_EXIST -from st2common.constants.error_messages import PACK_VIRTUALENV_USES_PYTHON3 from st2common.constants.system import API_URL_ENV_VARIABLE_NAME from st2common.constants.system import AUTH_TOKEN_ENV_VARIABLE_NAME from st2common.constants.triggers import (SENSOR_SPAWN_TRIGGER, SENSOR_EXIT_TRIGGER) @@ -44,7 +43,6 @@ from st2common.util.sandboxing import get_sandbox_python_path from st2common.util.sandboxing import get_sandbox_python_binary_path from st2common.util.sandboxing import get_sandbox_virtualenv_path -from st2common.util.sandboxing import is_pack_virtualenv_using_python3 __all__ = [ 'ProcessSensorContainer' @@ -297,14 +295,6 @@ def _spawn_sensor_process(self, sensor): msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values raise Exception(msg) - # NOTE: Running sensors using Python 3 virtual environments is not supported - uses_python3, _ = is_pack_virtualenv_using_python3(pack=sensor['pack']) - - if uses_python3 and not six.PY3: - format_values = {'pack': sensor['pack'], 'virtualenv_path': virtualenv_path} - msg = PACK_VIRTUALENV_USES_PYTHON3 % format_values - raise Exception(msg) - args = self._get_args_for_wrapper_script(python_binary=python_path, sensor=sensor) if self._enable_common_pack_libs: diff --git a/tools/config_gen.py b/tools/config_gen.py index 30769b3902..06e2050ac0 100755 --- a/tools/config_gen.py +++ b/tools/config_gen.py @@ -72,7 +72,6 @@ 'actionrunner': { 'virtualenv_binary': '/usr/bin/virtualenv', 'python_binary': '/usr/bin/python', - 'python3_binary': '/usr/bin/python3' }, 'webui': { 'webui_base_url': 'https://localhost' From cb73f03eb03a56f56ed3f09269fb7a817141cf58 Mon Sep 17 00:00:00 2001 From: amanda Date: Fri, 4 Dec 2020 13:03:18 +0000 Subject: [PATCH 02/10] Fix errors in UT amendments --- contrib/packs/tests/test_action_download.py | 437 ++++++++++++++++++- st2common/tests/unit/test_util_sandboxing.py | 36 ++ 2 files changed, 472 insertions(+), 1 deletion(-) diff --git a/contrib/packs/tests/test_action_download.py b/contrib/packs/tests/test_action_download.py index 02d940bd7f..3eeda00886 100644 --- a/contrib/packs/tests/test_action_download.py +++ b/contrib/packs/tests/test_action_download.py @@ -1,4 +1,439 @@ -def test_resolve_urls(self): +#!/usr/bin/env python + +# Copyright 2020 The StackStorm Authors. +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import mock +import shutil +import tempfile +import hashlib + +from st2common.util.monkey_patch import use_select_poll_workaround +use_select_poll_workaround() + +from lockfile import LockFile +from lockfile import LockTimeout +from git.repo import Repo +from gitdb.exc import BadName +from st2common.services import packs as pack_service +from st2tests.base import BaseActionTestCase + +import st2common.util.pack_management +from st2common.util.pack_management import eval_repo_url + +from pack_mgmt.download import DownloadGitRepoAction + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + +PACK_INDEX = { + "test": { + "version": "0.4.0", + "name": "test", + "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test", + "author": "st2-dev", + "keywords": ["some", "search", "another", "terms"], + "email": "info@stackstorm.com", + "description": "st2 pack to test package management pipeline" + }, + "test2": { + "version": "0.5.0", + "name": "test2", + "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test2", + "author": "stanley", + "keywords": ["some", "special", "terms"], + "email": "info@stackstorm.com", + "description": "another st2 pack to test package management pipeline" + }, + "test3": { + "version": "0.5.0", + "stackstorm_version": ">=1.6.0, <2.2.0", + "name": "test3", + "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test3", + "author": "stanley", + "keywords": ["some", "special", "terms"], + "email": "info@stackstorm.com", + "description": "another st2 pack to test package management pipeline" + }, + "test4": { + "version": "0.5.0", + "name": "test4", + "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test4", + "author": "stanley", + "keywords": ["some", "special", "terms"], "email": "info@stackstorm.com", + "description": "another st2 pack to test package management pipeline" + } +} + + +original_is_dir_func = os.path.isdir + + +def mock_is_dir_func(path): + """ + Mock function which returns True if path ends with .git + """ + if path.endswith('.git'): + return True + return original_is_dir_func(path) + + +def mock_get_gitref(repo, ref): + """ + Mock get_gitref function which return mocked object if ref passed is + PACK_INDEX['test']['version'] + """ + if PACK_INDEX['test']['version'] in ref: + if ref[0] == 'v': + return mock.MagicMock(hexsha=PACK_INDEX['test']['version']) + else: + return None + elif ref: + return mock.MagicMock(hexsha="abcDef") + else: + return None + + +@mock.patch.object(pack_service, 'fetch_pack_index', mock.MagicMock(return_value=(PACK_INDEX, {}))) +class DownloadGitRepoActionTestCase(BaseActionTestCase): + action_cls = DownloadGitRepoAction + + def setUp(self): + super(DownloadGitRepoActionTestCase, self).setUp() + + clone_from = mock.patch.object(Repo, 'clone_from') + + self.addCleanup(clone_from.stop) + self.clone_from = clone_from.start() + + self.expand_user_path = tempfile.mkdtemp() + expand_user = mock.patch.object(os.path, 'expanduser', + mock.MagicMock(return_value=self.expand_user_path)) + + self.addCleanup(expand_user.stop) + self.expand_user = expand_user.start() + + self.repo_base = tempfile.mkdtemp() + + self.repo_instance = mock.MagicMock() + type(self.repo_instance).active_branch = mock.Mock() + + def side_effect(url, to_path, **kwargs): + # Since we have no way to pass pack name here, we would have to derive it from repo url + fixture_name = url.split('/')[-1] + fixture_path = os.path.join(self._get_base_pack_path(), 'tests/fixtures', fixture_name) + shutil.copytree(fixture_path, to_path) + return self.repo_instance + + self.clone_from.side_effect = side_effect + + def tearDown(self): + shutil.rmtree(self.repo_base) + shutil.rmtree(self.expand_user()) + + def test_run_pack_download(self): + action = self.get_action_instance() + result = action.run(packs=['test'], abs_repo_base=self.repo_base) + temp_dir = hashlib.md5(PACK_INDEX['test']['repo_url'].encode()).hexdigest() + + self.assertEqual(result, {'test': 'Success.'}) + self.clone_from.assert_called_once_with(PACK_INDEX['test']['repo_url'], + os.path.join(os.path.expanduser('~'), temp_dir)) + self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test/pack.yaml'))) + + self.repo_instance.git.checkout.assert_called() + self.repo_instance.git.branch.assert_called() + self.repo_instance.git.checkout.assert_called() + + def test_run_pack_download_dependencies(self): + action = self.get_action_instance() + result = action.run(packs=['test'], dependency_list=['test2', 'test4'], + abs_repo_base=self.repo_base) + temp_dirs = [ + hashlib.md5(PACK_INDEX['test2']['repo_url'].encode()).hexdigest(), + hashlib.md5(PACK_INDEX['test4']['repo_url'].encode()).hexdigest() + ] + + self.assertEqual(result, {'test2': 'Success.', 'test4': 'Success.'}) + self.clone_from.assert_any_call(PACK_INDEX['test2']['repo_url'], + os.path.join(os.path.expanduser('~'), temp_dirs[0])) + self.clone_from.assert_any_call(PACK_INDEX['test4']['repo_url'], + os.path.join(os.path.expanduser('~'), temp_dirs[1])) + self.assertEqual(self.clone_from.call_count, 2) + self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test2/pack.yaml'))) + self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test4/pack.yaml'))) + + def test_run_pack_download_existing_pack(self): + action = self.get_action_instance() + action.run(packs=['test'], abs_repo_base=self.repo_base) + self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test/pack.yaml'))) + + result = action.run(packs=['test'], abs_repo_base=self.repo_base) + + self.assertEqual(result, {'test': 'Success.'}) + + def test_run_pack_download_multiple_packs(self): + action = self.get_action_instance() + result = action.run(packs=['test', 'test2'], abs_repo_base=self.repo_base) + temp_dirs = [ + hashlib.md5(PACK_INDEX['test']['repo_url'].encode()).hexdigest(), + hashlib.md5(PACK_INDEX['test2']['repo_url'].encode()).hexdigest() + ] + + self.assertEqual(result, {'test': 'Success.', 'test2': 'Success.'}) + self.clone_from.assert_any_call(PACK_INDEX['test']['repo_url'], + os.path.join(os.path.expanduser('~'), temp_dirs[0])) + self.clone_from.assert_any_call(PACK_INDEX['test2']['repo_url'], + os.path.join(os.path.expanduser('~'), temp_dirs[1])) + self.assertEqual(self.clone_from.call_count, 2) + self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test/pack.yaml'))) + self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test2/pack.yaml'))) + + @mock.patch.object(Repo, 'clone_from') + def test_run_pack_download_error(self, clone_from): + clone_from.side_effect = Exception('Something went terribly wrong during the clone') + + action = self.get_action_instance() + self.assertRaises(Exception, action.run, packs=['test'], abs_repo_base=self.repo_base) + + def test_run_pack_download_no_tag(self): + self.repo_instance.commit.side_effect = BadName + + action = self.get_action_instance() + self.assertRaises(ValueError, action.run, packs=['test=1.2.3'], + abs_repo_base=self.repo_base) + + def test_run_pack_lock_is_already_acquired(self): + action = self.get_action_instance() + temp_dir = hashlib.md5(PACK_INDEX['test']['repo_url'].encode()).hexdigest() + + original_acquire = LockFile.acquire + + def mock_acquire(self, timeout=None): + original_acquire(self, timeout=0.1) + + LockFile.acquire = mock_acquire + + try: + lock_file = LockFile('/tmp/%s' % (temp_dir)) + + # Acquire a lock (file) so acquire inside download will fail + with open(lock_file.lock_file, 'w') as fp: + fp.write('') + + expected_msg = 'Timeout waiting to acquire lock for' + self.assertRaisesRegexp(LockTimeout, expected_msg, action.run, packs=['test'], + abs_repo_base=self.repo_base) + finally: + os.unlink(lock_file.lock_file) + LockFile.acquire = original_acquire + + def test_run_pack_lock_is_already_acquired_force_flag(self): + # Lock is already acquired but force is true so it should be deleted and released + action = self.get_action_instance() + temp_dir = hashlib.md5(PACK_INDEX['test']['repo_url'].encode()).hexdigest() + + original_acquire = LockFile.acquire + + def mock_acquire(self, timeout=None): + original_acquire(self, timeout=0.1) + + LockFile.acquire = mock_acquire + + try: + lock_file = LockFile('/tmp/%s' % (temp_dir)) + + # Acquire a lock (file) so acquire inside download will fail + with open(lock_file.lock_file, 'w') as fp: + fp.write('') + + result = action.run(packs=['test'], abs_repo_base=self.repo_base, force=True) + finally: + LockFile.acquire = original_acquire + + self.assertEqual(result, {'test': 'Success.'}) + + def test_run_pack_download_v_tag(self): + def side_effect(ref): + if ref[0] != 'v': + raise BadName() + return mock.MagicMock(hexsha='abcdef') + + self.repo_instance.commit.side_effect = side_effect + self.repo_instance.git = mock.MagicMock( + branch=(lambda *args: 'master'), + checkout=(lambda *args: True) + ) + + action = self.get_action_instance() + result = action.run(packs=['test=1.2.3'], abs_repo_base=self.repo_base) + + self.assertEqual(result, {'test': 'Success.'}) + + @mock.patch.object(st2common.util.pack_management, 'get_valid_versions_for_repo', + mock.Mock(return_value=['1.0.0', '2.0.0'])) + def test_run_pack_download_invalid_version(self): + self.repo_instance.commit.side_effect = lambda ref: None + + action = self.get_action_instance() + + expected_msg = ('is not a valid version, hash, tag or branch.*?' + 'Available versions are: 1.0.0, 2.0.0.') + self.assertRaisesRegexp(ValueError, expected_msg, action.run, + packs=['test=2.2.3'], abs_repo_base=self.repo_base) + + def test_download_pack_stackstorm_version_identifier_check(self): + action = self.get_action_instance() + + # Version is satisfied + st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '2.0.0' + + result = action.run(packs=['test3'], abs_repo_base=self.repo_base) + self.assertEqual(result['test3'], 'Success.') + + # Pack requires a version which is not satisfied by current StackStorm version + st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '2.2.0' + expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' + 'current version is "2.2.0"') + self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], + abs_repo_base=self.repo_base) + + st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '2.3.0' + expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' + 'current version is "2.3.0"') + self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], + abs_repo_base=self.repo_base) + + st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '1.5.9' + expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' + 'current version is "1.5.9"') + self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], + abs_repo_base=self.repo_base) + + st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '1.5.0' + expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' + 'current version is "1.5.0"') + self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], + abs_repo_base=self.repo_base) + + # Version is not met, but force=true parameter is provided + st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '1.5.0' + result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=True) + self.assertEqual(result['test3'], 'Success.') + + def test_download_pack_python_version_check(self): + action = self.get_action_instance() + + # No python_versions attribute specified in the metadata file + with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ + mock_get_pack_metadata: + mock_get_pack_metadata.return_value = { + 'name': 'test3', + 'stackstorm_version': '', + 'python_versions': [] + } + + st2common.util.pack_management.six.PY2 = True + st2common.util.pack_management.six.PY3 = False + st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.11' + + result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False) + self.assertEqual(result['test3'], 'Success.') + + # Pack works with Python 2.x installation is running 2.7 + with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ + mock_get_pack_metadata: + mock_get_pack_metadata.return_value = { + 'name': 'test3', + 'stackstorm_version': '', + 'python_versions': ['2'] + } + + st2common.util.pack_management.six.PY2 = True + st2common.util.pack_management.six.PY3 = False + st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5' + + result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False) + self.assertEqual(result['test3'], 'Success.') + + st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.12' + + result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False) + self.assertEqual(result['test3'], 'Success.') + + # Pack works with Python 2.x installation is running 3.5 + with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ + mock_get_pack_metadata: + mock_get_pack_metadata.return_value = { + 'name': 'test3', + 'stackstorm_version': '', + 'python_versions': ['2'] + } + + st2common.util.pack_management.six.PY2 = False + st2common.util.pack_management.six.PY3 = True + + st2common.util.pack_management.CURRENT_PYTHON_VERSION = '3.5.2' + + expected_msg = (r'Pack "test3" requires Python 2.x, but current Python version is ' + '"3.5.2"') + self.assertRaisesRegexp(ValueError, expected_msg, action.run, + packs=['test3'], abs_repo_base=self.repo_base, force=False) + + # Pack works with Python 3.x installation is running 2.7 + with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ + mock_get_pack_metadata: + mock_get_pack_metadata.return_value = { + 'name': 'test3', + 'stackstorm_version': '', + 'python_versions': ['3'] + } + + st2common.util.pack_management.six.PY2 = True + st2common.util.pack_management.six.PY3 = False + st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.2' + + expected_msg = (r'Pack "test3" requires Python 3.x, but current Python version is ' + '"2.7.2"') + self.assertRaisesRegexp(ValueError, expected_msg, action.run, + packs=['test3'], abs_repo_base=self.repo_base, force=False) + + # Pack works with Python 2.x and 3.x installation is running 2.7 and 3.6.1 + with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ + mock_get_pack_metadata: + mock_get_pack_metadata.return_value = { + 'name': 'test3', + 'stackstorm_version': '', + 'python_versions': ['2', '3'] + } + + st2common.util.pack_management.six.PY2 = True + st2common.util.pack_management.six.PY3 = False + st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5' + + result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False) + self.assertEqual(result['test3'], 'Success.') + + st2common.util.pack_management.six.PY2 = False + st2common.util.pack_management.six.PY3 = True + st2common.util.pack_management.CURRENT_PYTHON_VERSION = '3.6.1' + + result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False) + self.assertEqual(result['test3'], 'Success.') + + def test_resolve_urls(self): url = eval_repo_url( "https://github.com/StackStorm-Exchange/stackstorm-test") self.assertEqual(url, "https://github.com/StackStorm-Exchange/stackstorm-test") diff --git a/st2common/tests/unit/test_util_sandboxing.py b/st2common/tests/unit/test_util_sandboxing.py index 8a713740cb..afda222e00 100644 --- a/st2common/tests/unit/test_util_sandboxing.py +++ b/st2common/tests/unit/test_util_sandboxing.py @@ -105,3 +105,39 @@ def test_get_sandbox_python_path(self, mock_get_python_lib): inherit_parent_virtualenv=True) self.assertEqual(python_path, ':/data/test1:/data/test2:%s/virtualenvtest' % (sys.prefix)) + + @mock.patch('os.path.isdir', mock.Mock(return_value=True)) + @mock.patch('os.listdir', mock.Mock(return_value=['python2.7'])) + @mock.patch('st2common.util.sandboxing.get_python_lib') + def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self, + mock_get_python_lib): + + # No inheritance + python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', + inherit_from_parent=False, + inherit_parent_virtualenv=False) + + self.assertEqual(python_path, ':') + + # Inherit python path from current process + # Mock the current process python path + os.environ['PYTHONPATH'] = ':/data/test1:/data/test2' + + python_path = get_sandbox_python_path(inherit_from_parent=True, + inherit_parent_virtualenv=False) + self.assertEqual(python_path, ':/data/test1:/data/test2') + + # Inherit from current process and from virtualenv (not running inside virtualenv) + del sys.real_prefix + + python_path = get_sandbox_python_path(inherit_from_parent=True, + inherit_parent_virtualenv=False) + self.assertEqual(python_path, ':/data/test1:/data/test2') + + # Inherit from current process and from virtualenv (running inside virtualenv) + sys.real_prefix = '/usr' + mock_get_python_lib.return_value = sys.prefix + '/virtualenvtest' + python_path = get_sandbox_python_path(inherit_from_parent=True, + inherit_parent_virtualenv=True) + self.assertEqual(python_path, ':/data/test1:/data/test2:%s/virtualenvtest' % + (sys.prefix)) From 4e8c626f780d23d852de20360ccc20f2ebf3385b Mon Sep 17 00:00:00 2001 From: amanda Date: Fri, 4 Dec 2020 15:02:14 +0000 Subject: [PATCH 03/10] Minor comment updates --- st2common/st2common/util/sandboxing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 7691cdc9e4..65ae17dd7c 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -132,7 +132,7 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, Return sandbox PYTHONPATH for a particular Python runner action. Same as get_sandbox_python_path() function, but it's intended to be used for Python runner - actions and also takes into account if a pack virtual environment uses Python 3. + actions. """ sandbox_python_path = get_sandbox_python_path( inherit_from_parent=inherit_from_parent, From de323e37dbdd3760d4dd6f25a09fa96d720cb635 Mon Sep 17 00:00:00 2001 From: amanda Date: Mon, 14 Dec 2020 14:38:15 +0000 Subject: [PATCH 04/10] Update changelog --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1edeb2b090..55d75163c1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -40,6 +40,9 @@ Fixed Removed ~~~~~~~~ +* Removed --python3 pack install option #5100 + Contributed by @amanda11 + * Removed check-licence script (cleanup) #5092 Contributed by @kroustou From 70cca5779c97696eb02de8634414c04ddb77ea7f Mon Sep 17 00:00:00 2001 From: Amanda McGuinness Date: Mon, 28 Dec 2020 14:53:33 +0000 Subject: [PATCH 05/10] Keep --python3 but omit warning --- st2client/st2client/commands/pack.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/st2client/st2client/commands/pack.py b/st2client/st2client/commands/pack.py index 617148642f..281326395f 100644 --- a/st2client/st2client/commands/pack.py +++ b/st2client/st2client/commands/pack.py @@ -189,6 +189,10 @@ def __init__(self, resource, *args, **kwargs): metavar='pack', help='Name of the %s in Exchange, or a git repo URL.' % resource.get_plural_display_name().lower()) + self.parser.add_argument('--python3', + action='store_true', + default=False, + help='Use Python 3 binary for pack virtual environment.') self.parser.add_argument('--force', action='store_true', default=False, @@ -206,6 +210,9 @@ def run(self, args, **kwargs): if not is_structured_output: self._get_content_counts_for_pack(args, **kwargs) + if args.python3: + print('\nDEPRECATION WARNING: --python3 flag will be ignored, as ST2 now runs with python3 as the default on all OS\n') + return self.manager.install(args.packs, force=args.force, skip_dependencies=args.skip_dependencies, **kwargs) From bc7cb401277d135311b7d4985fd449d98e323b63 Mon Sep 17 00:00:00 2001 From: Amanda McGuinness Date: Mon, 28 Dec 2020 16:59:04 +0000 Subject: [PATCH 06/10] Update deprecation warning message --- st2client/st2client/commands/pack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2client/st2client/commands/pack.py b/st2client/st2client/commands/pack.py index 281326395f..5e30bf8d76 100644 --- a/st2client/st2client/commands/pack.py +++ b/st2client/st2client/commands/pack.py @@ -211,7 +211,7 @@ def run(self, args, **kwargs): self._get_content_counts_for_pack(args, **kwargs) if args.python3: - print('\nDEPRECATION WARNING: --python3 flag will be ignored, as ST2 now runs with python3 as the default on all OS\n') + print('\nDEPRECATION WARNING: --python3 flag is ignored and will be removed in v3.5.0 as StackStorm now runs with python3 only\n') return self.manager.install(args.packs, force=args.force, skip_dependencies=args.skip_dependencies, **kwargs) From 66c6e5d6805a6336d8ea2bfe276d6e038baad659 Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 28 Dec 2020 15:16:57 -0700 Subject: [PATCH 07/10] Address linting issues --- st2client/st2client/commands/pack.py | 3 ++- st2common/st2common/config.py | 1 - st2common/st2common/util/pack_management.py | 1 - st2common/st2common/util/sandboxing.py | 9 +-------- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/st2client/st2client/commands/pack.py b/st2client/st2client/commands/pack.py index 5e30bf8d76..7db4352655 100644 --- a/st2client/st2client/commands/pack.py +++ b/st2client/st2client/commands/pack.py @@ -211,7 +211,8 @@ def run(self, args, **kwargs): self._get_content_counts_for_pack(args, **kwargs) if args.python3: - print('\nDEPRECATION WARNING: --python3 flag is ignored and will be removed in v3.5.0 as StackStorm now runs with python3 only\n') + print('\nDEPRECATION WARNING: --python3 flag is ignored and will be removed ' + 'in v3.5.0 as StackStorm now runs with python3 only\n') return self.manager.install(args.packs, force=args.force, skip_dependencies=args.skip_dependencies, **kwargs) diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index c0f5148f1c..9200b9a4f6 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -19,7 +19,6 @@ import sys from oslo_config import cfg -from distutils.spawn import find_executable from st2common.constants.system import VERSION_STRING from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH diff --git a/st2common/st2common/util/pack_management.py b/st2common/st2common/util/pack_management.py index d12b9a5f6c..65fd8c61ac 100644 --- a/st2common/st2common/util/pack_management.py +++ b/st2common/st2common/util/pack_management.py @@ -27,7 +27,6 @@ import re import six -from oslo_config import cfg from git.repo import Repo from gitdb.exc import BadName, BadObject from lockfile import LockFile diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 65ae17dd7c..a9711a4949 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -22,13 +22,11 @@ import os import sys -import fnmatch from distutils.sysconfig import get_python_lib from oslo_config import cfg from st2common.constants.pack import SYSTEM_PACK_NAMES -from st2common.content.utils import get_pack_base_path __all__ = [ 'get_sandbox_python_binary_path', @@ -134,15 +132,10 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, Same as get_sandbox_python_path() function, but it's intended to be used for Python runner actions. """ - sandbox_python_path = get_sandbox_python_path( + return get_sandbox_python_path( inherit_from_parent=inherit_from_parent, inherit_parent_virtualenv=inherit_parent_virtualenv) - pack_base_path = get_pack_base_path(pack_name=pack) - virtualenv_path = get_sandbox_virtualenv_path(pack=pack) - - return sandbox_python_path - def get_sandbox_virtualenv_path(pack): """ From 96b29ea8105198cfd4102062c147398aa4cdb55d Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 28 Dec 2020 15:19:07 -0700 Subject: [PATCH 08/10] Use warnings.warn --- st2client/st2client/commands/pack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2client/st2client/commands/pack.py b/st2client/st2client/commands/pack.py index 7db4352655..44918f0d23 100644 --- a/st2client/st2client/commands/pack.py +++ b/st2client/st2client/commands/pack.py @@ -211,8 +211,8 @@ def run(self, args, **kwargs): self._get_content_counts_for_pack(args, **kwargs) if args.python3: - print('\nDEPRECATION WARNING: --python3 flag is ignored and will be removed ' - 'in v3.5.0 as StackStorm now runs with python3 only\n') + warnings.warn('\nDEPRECATION WARNING: --python3 flag is ignored and will be removed ' + 'in v3.5.0 as StackStorm now runs with python3 only\n') return self.manager.install(args.packs, force=args.force, skip_dependencies=args.skip_dependencies, **kwargs) From 579fad68e022053dc10c4be0a506255541f8c9cc Mon Sep 17 00:00:00 2001 From: Amanda McGuinness Date: Mon, 28 Dec 2020 23:19:17 +0000 Subject: [PATCH 09/10] Add missing import --- st2client/st2client/commands/pack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/st2client/st2client/commands/pack.py b/st2client/st2client/commands/pack.py index 44918f0d23..b8e0265554 100644 --- a/st2client/st2client/commands/pack.py +++ b/st2client/st2client/commands/pack.py @@ -19,6 +19,7 @@ import six import editor +import warnings import yaml from st2client.models import Config From 48c4150b4abe4901cffb187fa5fc3f3bfcd55747 Mon Sep 17 00:00:00 2001 From: Amanda McGuinness Date: Tue, 29 Dec 2020 00:10:07 +0000 Subject: [PATCH 10/10] Newlines not needed with warnings.warn --- st2client/st2client/commands/pack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2client/st2client/commands/pack.py b/st2client/st2client/commands/pack.py index b8e0265554..827db663df 100644 --- a/st2client/st2client/commands/pack.py +++ b/st2client/st2client/commands/pack.py @@ -212,8 +212,8 @@ def run(self, args, **kwargs): self._get_content_counts_for_pack(args, **kwargs) if args.python3: - warnings.warn('\nDEPRECATION WARNING: --python3 flag is ignored and will be removed ' - 'in v3.5.0 as StackStorm now runs with python3 only\n') + warnings.warn('DEPRECATION WARNING: --python3 flag is ignored and will be removed ' + 'in v3.5.0 as StackStorm now runs with python3 only') return self.manager.install(args.packs, force=args.force, skip_dependencies=args.skip_dependencies, **kwargs)