diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a235f0b642..8e014a3f9a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -48,9 +48,11 @@ Fixed Removed ~~~~~~~~ -* Removed submit-debug-info tool and the st2debug component #5103 +* Removed --python3 pack install option #5100 Contributed by @amanda11 +* Removed submit-debug-info tool and the st2debug component #5103 + * Removed check-licence script (cleanup) #5092 Contributed by @kroustou * Updated Makefile and CI to use Python 3 only, removing Python 2 (cleanup) #5090 diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index 6e0abb00ca..a8111f59af 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -14,10 +14,6 @@ actions_pool_size = 60 logging = /etc/st2/logging.actionrunner.conf # List of pip options to be passed to "pip install" command when installing pack dependencies into pack virtual environment. pip_opts = # comma separated list allowed here. -# Python 3 binary which will be used by Python actions for packs which use Python 3 virtual environment. -python3_binary = /usr/bin/python3 -# Prefix for Python 3 installation (e.g. /opt/python3.6). If not specified, it tries to find Python 3 libraries in /usr/lib and /usr/local/lib. -python3_prefix = None # Python binary which will be used by Python actions. python_binary = /usr/bin/python # Default log level to use for Python runner actions. Can be overriden on invocation basis using "log_level" runner parameter. diff --git a/contrib/packs/actions/download.yaml b/contrib/packs/actions/download.yaml index 575d1ff45a..0849616b65 100644 --- a/contrib/packs/actions/download.yaml +++ b/contrib/packs/actions/download.yaml @@ -22,11 +22,6 @@ description: "Set to True to force install the pack and skip StackStorm version compatibility check" required: false default: false - python3: - type: "boolean" - description: "True to use Python 3 binary for this pack." - required: false - default: false dependency_list: type: "array" description: "Dependency list that needs to be downloaded." diff --git a/contrib/packs/actions/install.meta.yaml b/contrib/packs/actions/install.meta.yaml index 5d85a6ca04..1b8d0d572a 100644 --- a/contrib/packs/actions/install.meta.yaml +++ b/contrib/packs/actions/install.meta.yaml @@ -27,11 +27,6 @@ description: "Set to True to force install the pack and skip StackStorm version compatibility check and also delete and ignore lock file if one exists." required: false default: false - python3: - type: "boolean" - description: "Use Python 3 binary when creating a virtualenv for this pack." - required: false - default: false skip_dependencies: type: "boolean" description: "Set to True to skip pack dependency installations." diff --git a/contrib/packs/actions/pack_mgmt/download.py b/contrib/packs/actions/pack_mgmt/download.py index afe1c4db20..b4d888630b 100644 --- a/contrib/packs/actions/pack_mgmt/download.py +++ b/contrib/packs/actions/pack_mgmt/download.py @@ -63,7 +63,7 @@ def __init__(self, config=None, action_service=None): if self.proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None): os.environ['no_proxy'] = self.no_proxy - def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False, + def run(self, packs, abs_repo_base, verifyssl=True, force=False, dependency_list=None): result = {} pack_url = None @@ -73,7 +73,7 @@ def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False, pack_result = download_pack(pack=pack_dependency, abs_repo_base=abs_repo_base, verify_ssl=verifyssl, force=force, proxy_config=self.proxy_config, force_permissions=True, - use_python3=python3, logger=self.logger) + logger=self.logger) pack_url, pack_ref, pack_result = pack_result result[pack_ref] = pack_result else: @@ -82,7 +82,6 @@ def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False, verify_ssl=verifyssl, force=force, proxy_config=self.proxy_config, force_permissions=True, - use_python3=python3, logger=self.logger) pack_url, pack_ref, pack_result = pack_result result[pack_ref] = pack_result diff --git a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py index d60b8bbfe7..23f8a75ef7 100644 --- a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py +++ b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py @@ -74,7 +74,7 @@ def __init__(self, config=None, action_service=None): if self.proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None): os.environ['no_proxy'] = self.no_proxy - def run(self, packs, update=False, python3=False, no_download=True): + def run(self, packs, update=False, no_download=True): """ :param packs: A list of packs to create the environment for. :type: packs: ``list`` @@ -85,7 +85,7 @@ def run(self, packs, update=False, python3=False, no_download=True): for pack_name in packs: setup_pack_virtualenv(pack_name=pack_name, update=update, logger=self.logger, - proxy_config=self.proxy_config, use_python3=python3, + proxy_config=self.proxy_config, no_download=no_download) message = ('Successfully set up virtualenv for the following packs: %s' % diff --git a/contrib/packs/actions/setup_virtualenv.yaml b/contrib/packs/actions/setup_virtualenv.yaml index 6625cc70c5..18d1b3df15 100644 --- a/contrib/packs/actions/setup_virtualenv.yaml +++ b/contrib/packs/actions/setup_virtualenv.yaml @@ -14,11 +14,6 @@ type: "boolean" default: false description: "Check this option if the virtual environment already exists and if you only want to perform an update and installation of new dependencies. If you don't check this option, the virtual environment will be destroyed then re-created. If you check this and the virtual environment doesn't exist, it will create it." - python3: - type: "boolean" - description: "Use Python 3 binary when creating a virtualenv for this pack." - required: false - default: false no_download: type: "boolean" description: "Use version of pip which is already on the system instead of downloading the latest version from PyPi when creating a virtual environment." diff --git a/contrib/packs/actions/workflows/install.yaml b/contrib/packs/actions/workflows/install.yaml index 3d78668832..a67a754146 100644 --- a/contrib/packs/actions/workflows/install.yaml +++ b/contrib/packs/actions/workflows/install.yaml @@ -7,7 +7,6 @@ input: - register - env - force - - python3 - skip_dependencies - timeout @@ -30,7 +29,6 @@ tasks: input: packs: <% ctx().packs %> force: <% ctx().force %> - python3: <% ctx().python3 %> dependency_list: <% ctx().dependency_list %> next: - when: <% succeeded() %> @@ -98,7 +96,6 @@ tasks: input: packs: <% ctx().packs_list %> env: <% ctx().env %> - python3: <% ctx().python3 %> timeout: <% ctx().timeout %> next: - when: <% succeeded() %> diff --git a/contrib/packs/tests/test_action_download.py b/contrib/packs/tests/test_action_download.py index a5d84929fb..3eeda00886 100644 --- a/contrib/packs/tests/test_action_download.py +++ b/contrib/packs/tests/test_action_download.py @@ -433,57 +433,6 @@ def test_download_pack_python_version_check(self): result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False) self.assertEqual(result['test3'], 'Success.') - # StackStorm is running under Python 2, Pack requires Python 3 and --python3 flag is used - with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ - mock_get_pack_metadata: - mock_get_pack_metadata.return_value = { - 'name': 'test3', - 'stackstorm_version': '', - 'python_versions': ['3'] - } - - st2common.util.pack_management.six.PY2 = True - st2common.util.pack_management.six.PY3 = False - st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5' - - result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False, - python3=True) - self.assertEqual(result['test3'], 'Success.') - - with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ - mock_get_pack_metadata: - mock_get_pack_metadata.return_value = { - 'name': 'test3', - 'stackstorm_version': '', - 'python_versions': ['2', '3'] - } - - st2common.util.pack_management.six.PY2 = True - st2common.util.pack_management.six.PY3 = False - st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5' - - result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False, - python3=True) - self.assertEqual(result['test3'], 'Success.') - - # StackStorm is running under Python 2, pack requires Python 3 and --python3 flag is used - with mock.patch('st2common.util.pack_management.get_pack_metadata') as \ - mock_get_pack_metadata: - mock_get_pack_metadata.return_value = { - 'name': 'test3', - 'stackstorm_version': '', - 'python_versions': ['2'] - } - - st2common.util.pack_management.six.PY2 = True - st2common.util.pack_management.six.PY3 = False - st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5' - - expected_msg = (r'Pack "test3" requires Python 2.x, but --python3 flag is used') - self.assertRaisesRegexp(ValueError, expected_msg, action.run, - packs=['test3'], abs_repo_base=self.repo_base, force=False, - python3=True) - def test_resolve_urls(self): url = eval_repo_url( "https://github.com/StackStorm-Exchange/stackstorm-test") diff --git a/st2api/st2api/controllers/v1/packs.py b/st2api/st2api/controllers/v1/packs.py index 129ccd5cda..6193a3f01f 100644 --- a/st2api/st2api/controllers/v1/packs.py +++ b/st2api/st2api/controllers/v1/packs.py @@ -98,7 +98,6 @@ class PackInstallController(ActionExecutionsControllerMixin): def post(self, pack_install_request, requester_user=None): parameters = { 'packs': pack_install_request.packs, - 'python3': pack_install_request.python3 } if pack_install_request.force: diff --git a/st2client/st2client/commands/pack.py b/st2client/st2client/commands/pack.py index 310795ae6b..827db663df 100644 --- a/st2client/st2client/commands/pack.py +++ b/st2client/st2client/commands/pack.py @@ -19,6 +19,7 @@ import six import editor +import warnings import yaml from st2client.models import Config @@ -210,7 +211,11 @@ def run(self, args, **kwargs): if not is_structured_output: self._get_content_counts_for_pack(args, **kwargs) - return self.manager.install(args.packs, python3=args.python3, force=args.force, + if args.python3: + warnings.warn('DEPRECATION WARNING: --python3 flag is ignored and will be removed ' + 'in v3.5.0 as StackStorm now runs with python3 only') + + return self.manager.install(args.packs, force=args.force, skip_dependencies=args.skip_dependencies, **kwargs) def _get_content_counts_for_pack(self, args, **kwargs): diff --git a/st2client/st2client/models/core.py b/st2client/st2client/models/core.py index e6fec6da70..d102d3158f 100644 --- a/st2client/st2client/models/core.py +++ b/st2client/st2client/models/core.py @@ -507,12 +507,11 @@ class AsyncRequest(Resource): class PackResourceManager(ResourceManager): @add_auth_token_to_kwargs_from_env - def install(self, packs, force=False, python3=False, skip_dependencies=False, **kwargs): + def install(self, packs, force=False, skip_dependencies=False, **kwargs): url = '/%s/install' % (self.resource.get_url_path_name()) payload = { 'packs': packs, 'force': force, - 'python3': python3, 'skip_dependencies': skip_dependencies } response = self.client.post(url, payload, **kwargs) diff --git a/st2common/st2common/cmd/download_pack.py b/st2common/st2common/cmd/download_pack.py index e24432ff89..b22a3f0467 100644 --- a/st2common/st2common/cmd/download_pack.py +++ b/st2common/st2common/cmd/download_pack.py @@ -41,8 +41,6 @@ def _register_cli_opts(): cfg.BoolOpt('force', default=False, help='True to force pack download and ignore download ' 'lock file if it exists.'), - cfg.BoolOpt('use-python3', default=False, - help='True to use Python3 binary.') ] do_register_cli_opts(cli_opts) @@ -57,15 +55,13 @@ def main(argv): packs = cfg.CONF.pack verify_ssl = cfg.CONF.verify_ssl force = cfg.CONF.force - use_python3 = cfg.CONF.use_python3 proxy_config = get_and_set_proxy_config() for pack in packs: LOG.info('Installing pack "%s"' % (pack)) result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, - proxy_config=proxy_config, force_permissions=True, - use_python3=use_python3) + proxy_config=proxy_config, force_permissions=True) # Raw pack name excluding the version pack_name = result[1] diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py index bef02152bd..861d0d4041 100644 --- a/st2common/st2common/cmd/install_pack.py +++ b/st2common/st2common/cmd/install_pack.py @@ -42,9 +42,6 @@ def _register_cli_opts(): cfg.BoolOpt('force', default=False, help='True to force pack installation and ignore install ' 'lock file if it exists.'), - cfg.BoolOpt('use-python3', default=False, - help='True to use Python3 binary when creating virtualenv ' - 'for this pack.'), ] do_register_cli_opts(cli_opts) @@ -59,7 +56,6 @@ def main(argv): packs = cfg.CONF.pack verify_ssl = cfg.CONF.verify_ssl force = cfg.CONF.force - use_python3 = cfg.CONF.use_python3 proxy_config = get_and_set_proxy_config() @@ -67,8 +63,7 @@ def main(argv): # 1. Download the pack LOG.info('Installing pack "%s"' % (pack)) result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, - proxy_config=proxy_config, force_permissions=True, - use_python3=use_python3) + proxy_config=proxy_config, force_permissions=True) # Raw pack name excluding the version pack_name = result[1] @@ -84,7 +79,7 @@ def main(argv): # 2. Setup pack virtual environment LOG.info('Setting up virtualenv for pack "%s"' % (pack_name)) setup_pack_virtualenv(pack_name=pack_name, update=False, logger=LOG, - proxy_config=proxy_config, use_python3=use_python3, + proxy_config=proxy_config, no_download=True) LOG.info('Successfully set up virtualenv for pack "%s"' % (pack_name)) diff --git a/st2common/st2common/cmd/setup_pack_virtualenv.py b/st2common/st2common/cmd/setup_pack_virtualenv.py index d82e924d5c..626bb389af 100644 --- a/st2common/st2common/cmd/setup_pack_virtualenv.py +++ b/st2common/st2common/cmd/setup_pack_virtualenv.py @@ -39,8 +39,6 @@ def _register_cli_opts(): 'you don\'t check this option, the virtual environment will be destroyed ' 'then re-created. If you check this and the virtual environment doesn\'t ' 'exist, it will create it..')), - cfg.BoolOpt('python3', default=False, - help='Use Python 3 binary when creating a virtualenv for this pack.'), ] do_register_cli_opts(cli_opts) @@ -54,7 +52,6 @@ def main(argv): packs = cfg.CONF.pack update = cfg.CONF.update - use_python3 = cfg.CONF.python3 proxy_config = get_and_set_proxy_config() @@ -62,7 +59,7 @@ def main(argv): # Setup pack virtual environment LOG.info('Setting up virtualenv for pack "%s"' % (pack)) setup_pack_virtualenv(pack_name=pack, update=update, logger=LOG, - proxy_config=proxy_config, use_python3=use_python3, + proxy_config=proxy_config, no_download=True) LOG.info('Successfully set up virtualenv for pack "%s"' % (pack)) diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index d02ded345d..9200b9a4f6 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -19,7 +19,6 @@ import sys from oslo_config import cfg -from distutils.spawn import find_executable from st2common.constants.system import VERSION_STRING from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH @@ -345,7 +344,6 @@ def register_opts(ignore_errors=False): # Runner options default_python_bin_path = sys.executable - default_python3_bin_path = find_executable('python3') base_dir = os.path.dirname(os.path.realpath(default_python_bin_path)) default_virtualenv_bin_path = os.path.join(base_dir, 'virtualenv') @@ -359,14 +357,6 @@ def register_opts(ignore_errors=False): cfg.StrOpt( 'python_binary', default=default_python_bin_path, help='Python binary which will be used by Python actions.'), - cfg.StrOpt( - 'python3_binary', default=default_python3_bin_path, - help='Python 3 binary which will be used by Python actions for packs which ' - 'use Python 3 virtual environment.'), - cfg.StrOpt( - 'python3_prefix', default=None, - help='Prefix for Python 3 installation (e.g. /opt/python3.6). If not specified, it ' - 'tries to find Python 3 libraries in /usr/lib and /usr/local/lib.'), cfg.StrOpt( 'virtualenv_binary', default=default_virtualenv_bin_path, help='Virtualenv binary which should be used to create pack virtualenvs.'), diff --git a/st2common/st2common/constants/error_messages.py b/st2common/st2common/constants/error_messages.py index 749f7df659..7aa56c4025 100644 --- a/st2common/st2common/constants/error_messages.py +++ b/st2common/st2common/constants/error_messages.py @@ -15,7 +15,6 @@ __all__ = [ 'PACK_VIRTUALENV_DOESNT_EXIST', - 'PACK_VIRTUALENV_USES_PYTHON3', 'PYTHON2_DEPRECATION' ] @@ -26,14 +25,6 @@ "st2 run packs.setup_virtualenv packs=%(pack)s" ''' -PACK_VIRTUALENV_USES_PYTHON3 = ''' -Virtual environment (%(virtualenv_path)s) for pack "%(pack)s" is using Python 3. -Using Python 3 virtual environments in mixed deployments is only supported for Python runner -actions and not sensors. If you want to run this sensor, please re-recreate the -virtual environment with python2 binary: -"st2 run packs.setup_virtualenv packs=%(pack)s python3=false" -''' - PYTHON2_DEPRECATION = 'DEPRECATION WARNING. Support for python 2 will be removed in future ' \ 'StackStorm releases. Please ensure that all packs used are python ' \ '3 compatible. Your StackStorm installation may be upgraded from ' \ diff --git a/st2common/st2common/constants/pack.py b/st2common/st2common/constants/pack.py index f1e11d42d4..91ae5a5e2c 100644 --- a/st2common/st2common/constants/pack.py +++ b/st2common/st2common/constants/pack.py @@ -87,12 +87,6 @@ 'six>=1.9.0,<2.0' ] -# Python requirements which are common to all the packs and need to be installed -# for Python 3 pack virtual environments to work -BASE_PACK_PYTHON3_REQUIREMENTS = [ - 'pyyaml>=5.1,<5.2' -] - # Name of the pack manifest file MANIFEST_FILE_NAME = 'pack.yaml' diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index 3d6a1ec1f1..f1a116c3d6 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -5068,10 +5068,6 @@ definitions: type: array items: type: string - python3: - description: Use Python 3 binary for pack virtual environment. - type: boolean - default: false force: description: Force pack installation. type: boolean diff --git a/st2common/st2common/openapi.yaml.j2 b/st2common/st2common/openapi.yaml.j2 index 1d88c0018a..498e0e54a2 100644 --- a/st2common/st2common/openapi.yaml.j2 +++ b/st2common/st2common/openapi.yaml.j2 @@ -5064,10 +5064,6 @@ definitions: type: array items: type: string - python3: - description: Use Python 3 binary for pack virtual environment. - type: boolean - default: false force: description: Force pack installation. type: boolean diff --git a/st2common/st2common/util/pack_management.py b/st2common/st2common/util/pack_management.py index f435655e5c..65fd8c61ac 100644 --- a/st2common/st2common/util/pack_management.py +++ b/st2common/st2common/util/pack_management.py @@ -27,7 +27,6 @@ import re import six -from oslo_config import cfg from git.repo import Repo from gitdb.exc import BadName, BadObject from lockfile import LockFile @@ -71,7 +70,7 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, force=False, proxy_config=None, force_owner_group=True, force_permissions=True, - use_python3=False, logger=LOG): + logger=LOG): """ Download the pack and move it to /opt/stackstorm/packs. @@ -91,21 +90,11 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, :param force: Force the installation and ignore / delete the lock file if it already exists. :type force: ``bool`` - :param use_python3: True if a python3 binary should be used for this pack. - :type use_python3: ``bool`` - :return: (pack_url, pack_ref, result) :rtype: tuple """ proxy_config = proxy_config or {} - # Python3 binary check - python3_binary = cfg.CONF.actionrunner.python3_binary - if use_python3 and not python3_binary: - msg = ('Requested to use Python 3, but python3 binary not found on the system or ' - 'actionrunner.python3 config option is not configured correctly.') - raise ValueError(msg) - try: pack_url, pack_version = get_repo_url(pack, proxy_config=proxy_config) except Exception as e: @@ -162,7 +151,7 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, # 2. Verify that the pack version if compatible with current StackStorm version if not force: - verify_pack_version(pack_metadata=pack_metadata, use_python3=use_python3) + verify_pack_version(pack_metadata=pack_metadata) # 3. Move pack to the final location move_result = move_pack(abs_repo_base=abs_repo_base, pack_name=pack_ref, @@ -428,7 +417,7 @@ def is_desired_pack(abs_pack_path, pack_name): return (True, '') -def verify_pack_version(pack_metadata, use_python3=False): +def verify_pack_version(pack_metadata): """ Verify that the pack works with the currently running StackStorm version. """ @@ -447,17 +436,12 @@ def verify_pack_version(pack_metadata, use_python3=False): raise ValueError(msg) if supported_python_versions: - if set(supported_python_versions) == set(['2']) and (not six.PY2 or use_python3): - if use_python3: - msg = ('Pack "%s" requires Python 2.x, but --python3 flag is used. ' - 'You can override this restriction by providing the "force" flag, but ' - 'the pack is not guaranteed to work.' % (pack_name)) - else: - msg = ('Pack "%s" requires Python 2.x, but current Python version is "%s". ' - 'You can override this restriction by providing the "force" flag, but ' - 'the pack is not guaranteed to work.' % (pack_name, CURRENT_PYTHON_VERSION)) + if set(supported_python_versions) == set(['2']) and (not six.PY2): + msg = ('Pack "%s" requires Python 2.x, but current Python version is "%s". ' + 'You can override this restriction by providing the "force" flag, but ' + 'the pack is not guaranteed to work.' % (pack_name, CURRENT_PYTHON_VERSION)) raise ValueError(msg) - elif set(supported_python_versions) == set(['3']) and (not six.PY3 and not use_python3): + elif set(supported_python_versions) == set(['3']) and (not six.PY3): msg = ('Pack "%s" requires Python 3.x, but current Python version is "%s". ' 'You can override this restriction by providing the "force" flag, but ' 'the pack is not guaranteed to work.' % (pack_name, CURRENT_PYTHON_VERSION)) diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 930197b781..a9711a4949 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -22,13 +22,11 @@ import os import sys -import fnmatch from distutils.sysconfig import get_python_lib from oslo_config import cfg from st2common.constants.pack import SYSTEM_PACK_NAMES -from st2common.content.utils import get_pack_base_path __all__ = [ 'get_sandbox_python_binary_path', @@ -36,8 +34,6 @@ 'get_sandbox_python_path_for_python_action', 'get_sandbox_path', 'get_sandbox_virtualenv_path', - - 'is_pack_virtualenv_using_python3' ] @@ -134,100 +130,12 @@ def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, Return sandbox PYTHONPATH for a particular Python runner action. Same as get_sandbox_python_path() function, but it's intended to be used for Python runner - actions and also takes into account if a pack virtual environment uses Python 3. + actions. """ - sandbox_python_path = get_sandbox_python_path( + return get_sandbox_python_path( inherit_from_parent=inherit_from_parent, inherit_parent_virtualenv=inherit_parent_virtualenv) - pack_base_path = get_pack_base_path(pack_name=pack) - virtualenv_path = get_sandbox_virtualenv_path(pack=pack) - - if not virtualenv_path: - return sandbox_python_path - - uses_python3, virtualenv_directories = is_pack_virtualenv_using_python3(pack=pack) - if uses_python3: - # Add Python 3 lib directory (lib/python3.x) in front of the PYTHONPATH. This way we avoid - # issues with scripts trying to use packages / modules from Python 2.7 site-packages - # directory instead of the versions from Python 3 stdlib. - pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/') - pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') - python3_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0]) - - # Add Python 3 site-packages directory (lib/python3.x/site-packages) in front of the Python - # 2.7 system site-packages This is important because we want Python 3 compatible libraries - # to be used from the pack virtual environment and not system ones. - python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path, - virtualenv_directories[0], - 'site-packages') - - # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv - # one (e.g. /usr/lib/python3.6) - # NOTE: We can't simply use sys.prefix dir since it will be set to /opt/stackstorm/st2 - - system_prefix_dirs = [] - # Take custom prefix into account (if specified) - if cfg.CONF.actionrunner.python3_prefix: - system_prefix_dirs.append(cfg.CONF.actionrunner.python3_prefix) - - # By default, Python libs are installed either in /usr/lib/python3.x or - # /usr/local/lib/python3.x - system_prefix_dirs.extend(['/usr/lib', '/usr/local/lib']) - - for system_prefix_dir in system_prefix_dirs: - python3_system_lib_directory = os.path.join(system_prefix_dir, - virtualenv_directories[0]) - - if os.path.exists(python3_system_lib_directory): - break - - if not python3_system_lib_directory or not os.path.exists(python3_system_lib_directory): - python3_system_lib_directory = None - - full_sandbox_python_path = [] - - # NOTE: Order here is very important for imports to function correctly - if python3_system_lib_directory: - full_sandbox_python_path.append(python3_system_lib_directory) - - full_sandbox_python_path.append(python3_lib_directory) - full_sandbox_python_path.append(python3_site_packages_directory) - full_sandbox_python_path.append(pack_actions_lib_paths) - full_sandbox_python_path.append(sandbox_python_path) - - sandbox_python_path = ':'.join(full_sandbox_python_path) - - return sandbox_python_path - - -def is_pack_virtualenv_using_python3(pack): - """ - Return True if a particular pack virtual environment is using Python 3. - - :return: (uses_python3_bool, virtualenv_lib_directories) - :rtype: ``tuple`` - """ - # If python3.? directory exists in pack virtualenv lib/ path it means Python 3 is used by - # that virtual environment and we take that in to account when constructing PYTHONPATH - virtualenv_path = get_sandbox_virtualenv_path(pack=pack) - - if virtualenv_path and os.path.isdir(virtualenv_path): - pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') - - if not os.path.exists(pack_virtualenv_lib_path): - return False, None - - virtualenv_directories = os.listdir(pack_virtualenv_lib_path) - virtualenv_directories = [dir_name for dir_name in virtualenv_directories if - fnmatch.fnmatch(dir_name, 'python3*')] - uses_python3 = bool(virtualenv_directories) - else: - uses_python3 = False - virtualenv_directories = None - - return uses_python3, virtualenv_directories - def get_sandbox_virtualenv_path(pack): """ diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py index 88bdc49a73..a8e504ba4c 100644 --- a/st2common/st2common/util/virtualenvs.py +++ b/st2common/st2common/util/virtualenvs.py @@ -29,7 +29,6 @@ from st2common import log as logging from st2common.constants.pack import PACK_REF_WHITELIST_REGEX from st2common.constants.pack import BASE_PACK_REQUIREMENTS -from st2common.constants.pack import BASE_PACK_PYTHON3_REQUIREMENTS from st2common.util.shell import run_command from st2common.util.shell import quote_unix from st2common.util.compat import to_ascii @@ -46,7 +45,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True, include_setuptools=True, include_wheel=True, proxy_config=None, - use_python3=False, no_download=True, force_owner_group=True): + no_download=True, force_owner_group=True): """ Setup virtual environment for the provided pack. @@ -60,9 +59,6 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True :param logger: Optional logger instance to use. If not provided it defaults to the module level logger. - :param use_python3: Use Python3 binary when creating virtualenv for this pack. - :type use_python3: ``bool`` - :param no_download: Do not download and install latest version of pre-installed packages such as pip and distutils. :type no_download: ``bool`` @@ -95,7 +91,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path)) create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip, include_setuptools=include_setuptools, include_wheel=include_wheel, - use_python3=use_python3, no_download=no_download) + no_download=no_download) # 2. Install base requirements which are common to all the packs logger.debug('Installing base requirements') @@ -103,14 +99,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True install_requirement(virtualenv_path=virtualenv_path, requirement=requirement, proxy_config=proxy_config, logger=logger) - # 3. Install base Python 3 requirements which are common to all the packs - if use_python3: - logger.debug('Installing base Python 3 requirements') - for requirement in BASE_PACK_PYTHON3_REQUIREMENTS: - install_requirement(virtualenv_path=virtualenv_path, requirement=requirement, - proxy_config=proxy_config, logger=logger) - - # 4. Install pack-specific requirements + # 3. Install pack-specific requirements requirements_file_path = os.path.join(pack_path, 'requirements.txt') has_requirements = os.path.isfile(requirements_file_path) @@ -124,7 +113,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True else: logger.debug('No pack specific requirements found') - # 5. Set the owner group + # 4. Set the owner group if force_owner_group: apply_pack_owner_group(pack_path=virtualenv_path) @@ -134,7 +123,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_setuptools=True, - include_wheel=True, use_python3=False, no_download=True): + include_wheel=True, no_download=True): """ :param include_pip: Include pip binary and package in the newely created virtual environment. :type include_pip: ``bool`` @@ -146,9 +135,6 @@ def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_se :param include_wheel: Include wheel in the newely created virtual environment. :type include_wheel : ``bool`` - :param use_python3: Use Python3 binary when creating virtualenv for this pack. - :type use_python3: ``bool`` - :param no_download: Do not download and install latest version of pre-installed packages such as pip and distutils. :type no_download: ``bool`` @@ -157,7 +143,6 @@ def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_se logger = logger or LOG python_binary = cfg.CONF.actionrunner.python_binary - python3_binary = cfg.CONF.actionrunner.python3_binary virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts or [] @@ -172,15 +157,7 @@ def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_se cmd = [virtualenv_binary] - if use_python3 and not python3_binary: - msg = ('Requested to use Python 3, but python3 binary not found on the system or ' - 'actionrunner.python3 config option is not configured correctly.') - raise ValueError(msg) - - if use_python3: - cmd.extend(['-p', python3_binary]) - else: - cmd.extend(['-p', python_binary]) + cmd.extend(['-p', python_binary]) cmd.extend(virtualenv_opts) diff --git a/st2common/tests/unit/test_util_sandboxing.py b/st2common/tests/unit/test_util_sandboxing.py index 52b1fb4a44..afda222e00 100644 --- a/st2common/tests/unit/test_util_sandboxing.py +++ b/st2common/tests/unit/test_util_sandboxing.py @@ -26,7 +26,6 @@ from st2common.util.sandboxing import get_sandbox_python_path from st2common.util.sandboxing import get_sandbox_python_path_for_python_action from st2common.util.sandboxing import get_sandbox_python_binary_path -from st2common.util.sandboxing import is_pack_virtualenv_using_python3 import st2tests.config as tests_config @@ -112,7 +111,6 @@ def test_get_sandbox_python_path(self, mock_get_python_lib): @mock.patch('st2common.util.sandboxing.get_python_lib') def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self, mock_get_python_lib): - self.assertFalse(is_pack_virtualenv_using_python3(pack='dummy_pack')[0]) # No inheritance python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', @@ -143,93 +141,3 @@ def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self, inherit_parent_virtualenv=True) self.assertEqual(python_path, ':/data/test1:/data/test2:%s/virtualenvtest' % (sys.prefix)) - - @mock.patch('os.path.exists', mock.Mock(return_value=True)) - @mock.patch('os.path.isdir', mock.Mock(return_value=True)) - @mock.patch('os.listdir', mock.Mock(return_value=['python3.6'])) - @mock.patch('st2common.util.sandboxing.get_python_lib') - @mock.patch('st2common.util.sandboxing.get_pack_base_path', - mock.Mock(return_value='/tmp/packs/dummy_pack')) - @mock.patch('st2common.util.sandboxing.get_sandbox_virtualenv_path', - mock.Mock(return_value='/tmp/virtualenvs/dummy_pack')) - def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self, - mock_get_python_lib): - self.assertTrue(is_pack_virtualenv_using_python3(pack='dummy_pack')[0]) - - # No inheritance - python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', - inherit_from_parent=False, - inherit_parent_virtualenv=False) - - split = python_path.strip(':').split(':') - self.assertEqual(len(split), 4) - - # First entry should be system lib/python3 dir - self.assertIn('/usr/lib/python3.6', split[0]) - - # Second entry should be lib/python3 dir from venv - self.assertIn('virtualenvs/dummy_pack/lib/python3.6', split[1]) - - # Third entry should be python3 site-packages dir from venv - self.assertIn('virtualenvs/dummy_pack/lib/python3.6/site-packages', split[2]) - - # Fourth entry should be actions/lib dir from pack root directory - self.assertIn('packs/dummy_pack/actions/lib/', split[3]) - - # Inherit python path from current process - # Mock the current process python path - os.environ['PYTHONPATH'] = ':/data/test1:/data/test2' - - python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', - inherit_from_parent=True, - inherit_parent_virtualenv=False) - expected = ('/usr/lib/python3.6:' - '/tmp/virtualenvs/dummy_pack/lib/python3.6:' - '/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:' - '/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2') - self.assertEqual(python_path, expected) - - # Inherit from current process and from virtualenv (not running inside virtualenv) - del sys.real_prefix - - python_path = get_sandbox_python_path(inherit_from_parent=True, - inherit_parent_virtualenv=False) - self.assertEqual(python_path, ':/data/test1:/data/test2') - - # Inherit from current process and from virtualenv (running inside virtualenv) - sys.real_prefix = '/usr' - mock_get_python_lib.return_value = sys.prefix + '/virtualenvtest' - python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', - inherit_from_parent=True, - inherit_parent_virtualenv=True) - - expected = ('/usr/lib/python3.6:' - '/tmp/virtualenvs/dummy_pack/lib/python3.6:' - '/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:' - '/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2:' - '%s/virtualenvtest' % (sys.prefix)) - self.assertEqual(python_path, expected) - - # Custom prefix specified in the config - cfg.CONF.set_override(name='python3_prefix', override='/opt/lib', - group='actionrunner') - - # No inheritance - python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack', - inherit_from_parent=False, - inherit_parent_virtualenv=False) - - split = python_path.strip(':').split(':') - self.assertEqual(len(split), 4) - - # First entry should be system lib/python3 dir - self.assertIn('/opt/lib/python3.6', split[0]) - - # Second entry should be lib/python3 dir from venv - self.assertIn('virtualenvs/dummy_pack/lib/python3.6', split[1]) - - # Third entry should be python3 site-packages dir from venv - self.assertIn('virtualenvs/dummy_pack/lib/python3.6/site-packages', split[2]) - - # Fourth entry should be actions/lib dir from pack root directory - self.assertIn('packs/dummy_pack/actions/lib/', split[3]) diff --git a/st2common/tests/unit/test_virtualenvs.py b/st2common/tests/unit/test_virtualenvs.py index b7d6f37a3e..90c0f4e989 100644 --- a/st2common/tests/unit/test_virtualenvs.py +++ b/st2common/tests/unit/test_virtualenvs.py @@ -293,49 +293,6 @@ def test_install_requirements_with_https_proxy_no_cert(self): } virtualenvs.run_command.assert_called_once_with(**expected_args) - @mock.patch.object(virtualenvs, 'run_command') - def test_setup_pack_virtualenv_use_python3_binary(self, mock_run_command): - mock_run_command.return_value = 0, '', '' - - cfg.CONF.set_override(name='python_binary', group='actionrunner', - override='/usr/bin/python2.7') - cfg.CONF.set_override(name='python3_binary', group='actionrunner', - override='/usr/bin/python3') - - pack_name = 'dummy_pack_2' - - # Python 2 - setup_pack_virtualenv(pack_name=pack_name, update=False, - include_setuptools=False, include_wheel=False, - use_python3=False) - - actual_cmd = mock_run_command.call_args_list[0][1]['cmd'] - actual_cmd = ' '.join(actual_cmd) - - self.assertEqual(mock_run_command.call_count, 2) - self.assertIn('-p /usr/bin/python2.7', actual_cmd) - - mock_run_command.reset_mock() - - # Python 3 - setup_pack_virtualenv(pack_name=pack_name, update=False, - include_setuptools=False, include_wheel=False, - use_python3=True) - - self.assertEqual(mock_run_command.call_count, 3) - - actual_cmd = mock_run_command.call_args_list[0][1]['cmd'] - actual_cmd = ' '.join(actual_cmd) - self.assertIn('-p /usr/bin/python3', actual_cmd) - - actual_cmd = mock_run_command.call_args_list[1][1]['cmd'] - actual_cmd = ' '.join(actual_cmd) - self.assertIn('pip install pyyaml', actual_cmd) - - actual_cmd = mock_run_command.call_args_list[2][1]['cmd'] - actual_cmd = ' '.join(actual_cmd) - self.assertIn('pip install', actual_cmd) - def assertVirtualenvExists(self, virtualenv_dir): self.assertTrue(os.path.exists(virtualenv_dir)) self.assertTrue(os.path.isdir(virtualenv_dir)) diff --git a/st2reactor/st2reactor/container/process_container.py b/st2reactor/st2reactor/container/process_container.py index 34b9535063..f8f1638d71 100644 --- a/st2reactor/st2reactor/container/process_container.py +++ b/st2reactor/st2reactor/container/process_container.py @@ -29,7 +29,6 @@ from st2common import log as logging from st2common.util import concurrency from st2common.constants.error_messages import PACK_VIRTUALENV_DOESNT_EXIST -from st2common.constants.error_messages import PACK_VIRTUALENV_USES_PYTHON3 from st2common.constants.system import API_URL_ENV_VARIABLE_NAME from st2common.constants.system import AUTH_TOKEN_ENV_VARIABLE_NAME from st2common.constants.triggers import (SENSOR_SPAWN_TRIGGER, SENSOR_EXIT_TRIGGER) @@ -44,7 +43,6 @@ from st2common.util.sandboxing import get_sandbox_python_path from st2common.util.sandboxing import get_sandbox_python_binary_path from st2common.util.sandboxing import get_sandbox_virtualenv_path -from st2common.util.sandboxing import is_pack_virtualenv_using_python3 __all__ = [ 'ProcessSensorContainer' @@ -297,14 +295,6 @@ def _spawn_sensor_process(self, sensor): msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values raise Exception(msg) - # NOTE: Running sensors using Python 3 virtual environments is not supported - uses_python3, _ = is_pack_virtualenv_using_python3(pack=sensor['pack']) - - if uses_python3 and not six.PY3: - format_values = {'pack': sensor['pack'], 'virtualenv_path': virtualenv_path} - msg = PACK_VIRTUALENV_USES_PYTHON3 % format_values - raise Exception(msg) - args = self._get_args_for_wrapper_script(python_binary=python_path, sensor=sensor) if self._enable_common_pack_libs: diff --git a/tools/config_gen.py b/tools/config_gen.py index 30769b3902..06e2050ac0 100755 --- a/tools/config_gen.py +++ b/tools/config_gen.py @@ -72,7 +72,6 @@ 'actionrunner': { 'virtualenv_binary': '/usr/bin/virtualenv', 'python_binary': '/usr/bin/python', - 'python3_binary': '/usr/bin/python3' }, 'webui': { 'webui_base_url': 'https://localhost'