diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1aa027c819..6c50ca5499 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,7 @@ Added ``url_hosts_blacklist`` and ``url_hosts_whitelist`` runner attribute. (new feature) #4757 * Add ``user`` parameter to ``re_run`` method of st2client. #4785 +* Install pack dependencies automatically. #4769 Changed ~~~~~~~ diff --git a/contrib/core/actions/error.yaml b/contrib/core/actions/error.yaml new file mode 100644 index 0000000000..089aeb9737 --- /dev/null +++ b/contrib/core/actions/error.yaml @@ -0,0 +1,24 @@ +--- +description: Action that executes the Linux echo command (to stderr) on the localhost. +runner_type: "local-shell-cmd" +enabled: true +entry_point: '' +name: error +parameters: + message: + description: The message that the command will echo to stderr. + type: string + required: true + cmd: + description: Arbitrary Linux command to be executed on the local host. + required: true + type: string + default: '>&2 echo "{{message}}"' + immutable: true + kwarg_op: + immutable: true + sudo: + default: false + immutable: true + sudo_password: + immutable: true diff --git a/contrib/hello_st2/pack.yaml b/contrib/hello_st2/pack.yaml index 365ec13e40..afd34066c3 100644 --- a/contrib/hello_st2/pack.yaml +++ b/contrib/hello_st2/pack.yaml @@ -16,6 +16,12 @@ version: 0.1.0 python_versions: - "2" - "3" +# New in StackStorm 3.2 +# Specify a list of dependency packs to install. If the pack is in StackStorm Exchange you can use +# the pack name, or you can specify a full Git repository URL. Optionally, you can specify the +# exact version, tag, or branch. +dependencies: + - core # Name of the pack author. author: StackStorm, Inc. # Email of the pack author. diff --git a/contrib/packs/actions/delete.yaml b/contrib/packs/actions/delete.yaml index cb7e5f33e7..c8eb18f842 100644 --- a/contrib/packs/actions/delete.yaml +++ b/contrib/packs/actions/delete.yaml @@ -14,3 +14,8 @@ type: "string" default: "/opt/stackstorm/packs/" immutable: true + delete_env: + type: "boolean" + description: Delete virtual environment for pack when set to True. + default: true + required: false diff --git a/contrib/packs/actions/download.yaml b/contrib/packs/actions/download.yaml index 543db4a63c..575d1ff45a 100644 --- a/contrib/packs/actions/download.yaml +++ b/contrib/packs/actions/download.yaml @@ -27,3 +27,9 @@ description: "True to use Python 3 binary for this pack." required: false default: false + dependency_list: + type: "array" + description: "Dependency list that needs to be downloaded." + items: + type: "string" + required: false diff --git a/contrib/packs/actions/get_pack_dependencies.yaml b/contrib/packs/actions/get_pack_dependencies.yaml new file mode 100644 index 0000000000..45c7c01245 --- /dev/null +++ b/contrib/packs/actions/get_pack_dependencies.yaml @@ -0,0 +1,20 @@ + + +--- + name: "get_pack_dependencies" + runner_type: "python-script" + description: "Get pack dependencies specified in pack.yaml" + enabled: true + pack: packs + entry_point: "pack_mgmt/get_pack_dependencies.py" + parameters: + packs_status: + type: object + description: Name of the pack in Exchange or a git repo URL and download status. + required: true + default: null + nested: + type: integer + description: Nested level of dependencies to prevent infinite or really long download loops. + required: true + default: 3 diff --git a/contrib/packs/actions/install.meta.yaml b/contrib/packs/actions/install.meta.yaml index 46632de72a..1b6401ce4c 100644 --- a/contrib/packs/actions/install.meta.yaml +++ b/contrib/packs/actions/install.meta.yaml @@ -1,6 +1,6 @@ --- name: "install" - runner_type: "action-chain" + runner_type: "orquesta" description: "Installs or upgrades a pack into local content repository, either by git URL or a short name matching an index entry. Will download pack, load the actions, sensors and rules from the pack. @@ -32,3 +32,8 @@ description: "Use Python 3 binary when creating a virtualenv for this pack." required: false default: false + skip_dependencies: + type: "boolean" + description: "Set to True to skip pack dependency installations." + required: false + default: false diff --git a/contrib/packs/actions/pack_mgmt/delete.py b/contrib/packs/actions/pack_mgmt/delete.py index e023585b8c..570c16865a 100644 --- a/contrib/packs/actions/pack_mgmt/delete.py +++ b/contrib/packs/actions/pack_mgmt/delete.py @@ -30,7 +30,7 @@ def __init__(self, config=None, action_service=None): self._base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/') - def run(self, packs, abs_repo_base): + def run(self, packs, abs_repo_base, delete_env=True): intersection = BLOCKED_PACKS & frozenset(packs) if len(intersection) > 0: names = ', '.join(list(intersection)) @@ -43,12 +43,13 @@ def run(self, packs, abs_repo_base): self.logger.debug('Deleting pack directory "%s"' % (abs_fp)) shutil.rmtree(abs_fp) - # 2. Delete pack virtual environment - for pack_name in packs: - pack_name = quote_unix(pack_name) - virtualenv_path = os.path.join(self._base_virtualenvs_path, pack_name) + if delete_env: + # 2. Delete pack virtual environment + for pack_name in packs: + pack_name = quote_unix(pack_name) + virtualenv_path = os.path.join(self._base_virtualenvs_path, pack_name) - if os.path.isdir(virtualenv_path): - self.logger.debug('Deleting virtualenv "%s" for pack "%s"' % - (virtualenv_path, pack_name)) - shutil.rmtree(virtualenv_path) + if os.path.isdir(virtualenv_path): + self.logger.debug('Deleting virtualenv "%s" for pack "%s"' % + (virtualenv_path, pack_name)) + shutil.rmtree(virtualenv_path) diff --git a/contrib/packs/actions/pack_mgmt/download.py b/contrib/packs/actions/pack_mgmt/download.py index 344e3dfa66..f3c877f377 100644 --- a/contrib/packs/actions/pack_mgmt/download.py +++ b/contrib/packs/actions/pack_mgmt/download.py @@ -62,18 +62,29 @@ def __init__(self, config=None, action_service=None): if self.proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None): os.environ['no_proxy'] = self.no_proxy - def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False): + def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False, + dependency_list=None): result = {} - - for pack in packs: - pack_result = download_pack(pack=pack, abs_repo_base=abs_repo_base, - verify_ssl=verifyssl, force=force, - proxy_config=self.proxy_config, - force_permissions=True, - use_python3=python3, - logger=self.logger) - pack_url, pack_ref, pack_result = pack_result - result[pack_ref] = pack_result + pack_url = None + + if dependency_list: + for pack_dependency in dependency_list: + pack_result = download_pack(pack=pack_dependency, abs_repo_base=abs_repo_base, + verify_ssl=verifyssl, force=force, + proxy_config=self.proxy_config, force_permissions=True, + use_python3=python3, logger=self.logger) + pack_url, pack_ref, pack_result = pack_result + result[pack_ref] = pack_result + else: + for pack in packs: + pack_result = download_pack(pack=pack, abs_repo_base=abs_repo_base, + verify_ssl=verifyssl, force=force, + proxy_config=self.proxy_config, + force_permissions=True, + use_python3=python3, + logger=self.logger) + pack_url, pack_ref, pack_result = pack_result + result[pack_ref] = pack_result return self._validate_result(result=result, repo_url=pack_url) diff --git a/contrib/packs/actions/pack_mgmt/get_pack_dependencies.py b/contrib/packs/actions/pack_mgmt/get_pack_dependencies.py new file mode 100644 index 0000000000..b28776e182 --- /dev/null +++ b/contrib/packs/actions/pack_mgmt/get_pack_dependencies.py @@ -0,0 +1,131 @@ +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function + +import six + +from st2common.constants.pack import PACK_VERSION_SEPARATOR +from st2common.content.utils import get_pack_base_path +from st2common.runners.base_action import Action +from st2common.util.pack import get_pack_metadata + + +class GetPackDependencies(Action): + def run(self, packs_status, nested): + """ + :param packs_status: Name of the pack in Exchange or a git repo URL and download status. + :type: packs_status: ``dict`` + + :param nested: Nested level of dependencies to prevent infinite or really + long download loops. + :type nested: ``integer`` + """ + result = {} + dependency_list = [] + conflict_list = [] + + if not packs_status or nested == 0: + return result + + for pack, status in six.iteritems(packs_status): + if 'success' not in status.lower(): + continue + + dependency_packs = get_dependency_list(pack) + if not dependency_packs: + continue + + for dep_pack in dependency_packs: + name_or_url, pack_version = self.get_name_and_version(dep_pack) + + if len(name_or_url.split('/')) == 1: + pack_name = name_or_url + else: + name_or_git = name_or_url.split("/")[-1] + pack_name = name_or_git if '.git' not in name_or_git else \ + name_or_git.split('.')[0] + + # Check existing pack by pack name + existing_pack_version = get_pack_version(pack_name) + + # Try one more time to get existing pack version by name if 'stackstorm-' is in + # pack name + if not existing_pack_version and 'stackstorm-' in pack_name.lower(): + existing_pack_version = get_pack_version(pack_name.split('stackstorm-')[-1]) + + if existing_pack_version: + if existing_pack_version and not existing_pack_version.startswith('v'): + existing_pack_version = 'v' + existing_pack_version + if pack_version and not pack_version.startswith('v'): + pack_version = 'v' + pack_version + if pack_version and existing_pack_version != pack_version \ + and dep_pack not in conflict_list: + conflict_list.append(dep_pack) + else: + conflict = self.check_dependency_list_for_conflict(name_or_url, pack_version, + dependency_list) + if conflict: + conflict_list.append(dep_pack) + elif dep_pack not in dependency_list: + dependency_list.append(dep_pack) + + result['dependency_list'] = dependency_list + result['conflict_list'] = conflict_list + result['nested'] = nested - 1 + + return result + + def check_dependency_list_for_conflict(self, name, version, dependency_list): + conflict = False + + for pack in dependency_list: + name_or_url, pack_version = self.get_name_and_version(pack) + if name == name_or_url: + if version != pack_version: + conflict = True + break + + return conflict + + @staticmethod + def get_name_and_version(pack): + pack_and_version = pack.split(PACK_VERSION_SEPARATOR) + name_or_url = pack_and_version[0] + pack_version = pack_and_version[1] if len(pack_and_version) > 1 else None + + return name_or_url, pack_version + + +def get_pack_version(pack=None): + pack_path = get_pack_base_path(pack) + try: + pack_metadata = get_pack_metadata(pack_dir=pack_path) + result = pack_metadata.get('version', None) + except Exception: + result = None + finally: + return result + + +def get_dependency_list(pack=None): + pack_path = get_pack_base_path(pack) + + try: + pack_metadata = get_pack_metadata(pack_dir=pack_path) + result = pack_metadata.get('dependencies', None) + except Exception: + print('Could not open pack.yaml at location %s' % pack_path) + result = None + finally: + return result diff --git a/contrib/packs/actions/pack_mgmt/register.py b/contrib/packs/actions/pack_mgmt/register.py index d59ba21513..d79324a55d 100644 --- a/contrib/packs/actions/pack_mgmt/register.py +++ b/contrib/packs/actions/pack_mgmt/register.py @@ -72,6 +72,7 @@ def run(self, register, packs=None): 'types': types } + packs.reverse() if packs: method_kwargs['packs'] = packs diff --git a/contrib/packs/actions/pack_mgmt/virtualenv_setup_prerun.py b/contrib/packs/actions/pack_mgmt/virtualenv_setup_prerun.py index 5801c4e83a..91a7731128 100644 --- a/contrib/packs/actions/pack_mgmt/virtualenv_setup_prerun.py +++ b/contrib/packs/actions/pack_mgmt/virtualenv_setup_prerun.py @@ -18,13 +18,22 @@ class PacksTransformationAction(Action): - def run(self, packs_status): + def run(self, packs_status, packs_list=None): """ :param packs_status: Result from packs.download action. :type: packs_status: ``dict`` + + :param packs_list: Names of the pack in Exchange, a git repo URL or local file system. + :type: packs_list: ``list`` """ + if not packs_list: + packs_list = [] + packs = [] for pack_name, status in six.iteritems(packs_status): if 'success' in status.lower(): packs.append(pack_name) - return packs + + packs_list.extend(packs) + + return packs_list diff --git a/contrib/packs/actions/virtualenv_prerun.yaml b/contrib/packs/actions/virtualenv_prerun.yaml index f65bf198cb..74448d2937 100644 --- a/contrib/packs/actions/virtualenv_prerun.yaml +++ b/contrib/packs/actions/virtualenv_prerun.yaml @@ -7,3 +7,10 @@ parameters: packs_status: type: "object" + packs_list: + type: array + items: + type: string + required: false + default: null + description: Names of the pack in Exchange, a git repo URL or local file system. diff --git a/contrib/packs/actions/workflows/install.yaml b/contrib/packs/actions/workflows/install.yaml index e1412fb5ee..ceb8091ce3 100644 --- a/contrib/packs/actions/workflows/install.yaml +++ b/contrib/packs/actions/workflows/install.yaml @@ -1,32 +1,116 @@ ---- - chain: - - - name: "download pack" - ref: "packs.download" - parameters: - packs: "{{packs}}" - force: "{{force}}" - python3: "{{python3}}" - on-success: "make a prerun" - - - name: "make a prerun" - ref: "packs.virtualenv_prerun" - parameters: - packs_status: "{{ __results['download pack'].result }}" - on-success: "install pack dependencies" - - - name: "install pack dependencies" - ref: "packs.setup_virtualenv" - parameters: - packs: "{{ __results['make a prerun'].result }}" - env: "{{env}}" - python3: "{{python3}}" - on-success: "register pack" - - - name: "register pack" - ref: "packs.load" - parameters: - register: "{{register}}" - packs: "{{ __results['make a prerun'].result }}" - - default: "download pack" +version: 1.0 + +description: A orquesta workflow to install packs. + +input: + - packs + - register + - env + - force + - python3 + - skip_dependencies + +vars: + - packs_list: null + - dependency_list: null + - conflict_list: null + - nested: 10 + - message: "" + +tasks: + init_task: + action: core.noop + next: + - do: download_pack + + download_pack: + action: packs.download + input: + packs: <% ctx().packs %> + force: <% ctx().force %> + python3: <% ctx().python3 %> + dependency_list: <% ctx().dependency_list %> + next: + - when: <% succeeded() %> + do: make_a_prerun + + make_a_prerun: + action: packs.virtualenv_prerun + input: + packs_status: <% task(download_pack).result.result %> + packs_list: <% ctx().packs_list %> + next: + - when: <% succeeded() and (ctx().skip_dependencies or ctx().nested = 0) %> + publish: + - packs_list: <% task(make_a_prerun).result.result %> + do: install_pack_requirements + - when: <% succeeded() and (ctx().nested > 0 and not ctx().skip_dependencies) %> + publish: + - packs_list: <% task(make_a_prerun).result.result %> + do: get_pack_dependencies + + get_pack_dependencies: + action: packs.get_pack_dependencies + input: + packs_status: <% task(download_pack).result.result %> + nested: <% ctx().nested%> + next: + - when: <% succeeded() %> + publish: + - dependency_list: <% result().result.dependency_list %> + - conflict_list: <% result().result.conflict_list %> + - nested: <% result().result.nested %> + do: check_dependency_and_conflict_list + + check_dependency_and_conflict_list: + action: core.noop + next: + - when: <% ctx().conflict_list %> + do: stop_installation_and_cleanup_because_conflict + - when: <% not ctx().conflict_list and ctx().dependency_list %> + do: download_pack + - when: <% not ctx().conflict_list and not ctx().dependency_list %> + do: install_pack_requirements + + stop_installation_and_cleanup_because_conflict: + action: packs.delete + input: + packs: <% ctx().packs_list %> + delete_env: false + next: + - do: echo_pack_conflicts + + echo_pack_conflicts: + action: core.noop + next: + - publish: + - message: >- + Unable to install packs due to conflicts. Review the + conflict_list and check the versions of corresponding installed + packs. You can also run the `st2 pack install` command with the + `--skip-dependencies` flag to skip installing dependent packs. + do: fail + + install_pack_requirements: + action: packs.setup_virtualenv + input: + packs: <% ctx().packs_list %> + env: <% ctx().env %> + python3: <% ctx().python3 %> + next: + - when: <% succeeded() %> + do: register_pack + + register_pack: + action: packs.load + input: + register: <% ctx().register %> + packs: <% ctx().packs_list %> + next: + - publish: + - message: Successfully installed packs + +output: + - packs_list: <% ctx().packs_list %> + - message: <% ctx().message %> + - conflict_list: <% ctx().conflict_list %> diff --git a/contrib/packs/pack.yaml b/contrib/packs/pack.yaml index aa07727c4c..531bd24c34 100644 --- a/contrib/packs/pack.yaml +++ b/contrib/packs/pack.yaml @@ -1,7 +1,7 @@ --- name : packs description : Pack management functionality. -version : 1.0.1 +version : 2.0.0 python_versions: - "2" - "3" diff --git a/contrib/packs/tests/test_action_download.py b/contrib/packs/tests/test_action_download.py index 7d0becc2b3..2b5b3bc828 100644 --- a/contrib/packs/tests/test_action_download.py +++ b/contrib/packs/tests/test_action_download.py @@ -65,6 +65,14 @@ "keywords": ["some", "special", "terms"], "email": "info@stackstorm.com", "description": "another st2 pack to test package management pipeline" + }, + "test4": { + "version": "0.5.0", + "name": "test4", + "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test4", + "author": "stanley", + "keywords": ["some", "special", "terms"], "email": "info@stackstorm.com", + "description": "another st2 pack to test package management pipeline" } } @@ -148,6 +156,24 @@ def test_run_pack_download(self): self.repo_instance.git.branch.assert_called() self.repo_instance.git.checkout.assert_called() + def test_run_pack_download_dependencies(self): + action = self.get_action_instance() + result = action.run(packs=['test'], dependency_list=['test2', 'test4'], + abs_repo_base=self.repo_base) + temp_dirs = [ + hashlib.md5(PACK_INDEX['test2']['repo_url'].encode()).hexdigest(), + hashlib.md5(PACK_INDEX['test4']['repo_url'].encode()).hexdigest() + ] + + self.assertEqual(result, {'test2': 'Success.', 'test4': 'Success.'}) + self.clone_from.assert_any_call(PACK_INDEX['test2']['repo_url'], + os.path.join(os.path.expanduser('~'), temp_dirs[0])) + self.clone_from.assert_any_call(PACK_INDEX['test4']['repo_url'], + os.path.join(os.path.expanduser('~'), temp_dirs[1])) + self.assertEqual(self.clone_from.call_count, 2) + self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test2/pack.yaml'))) + self.assertTrue(os.path.isfile(os.path.join(self.repo_base, 'test4/pack.yaml'))) + def test_run_pack_download_existing_pack(self): action = self.get_action_instance() action.run(packs=['test'], abs_repo_base=self.repo_base) diff --git a/contrib/packs/tests/test_get_pack_dependencies.py b/contrib/packs/tests/test_get_pack_dependencies.py new file mode 100644 index 0000000000..6df299bb3e --- /dev/null +++ b/contrib/packs/tests/test_get_pack_dependencies.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python + +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from st2tests.base import BaseActionTestCase + +from pack_mgmt.get_pack_dependencies import GetPackDependencies + +UNINSTALLED_PACK = 'uninstalled_pack' +UNINSTALLED_PACKS = [ + UNINSTALLED_PACK, + 'https://github.com/StackStorm-Exchange/stackstorm-pack1', + 'https://github.com/StackStorm-Exchange/stackstorm-pack2.git', + 'https://github.com/StackStorm-Exchange/stackstorm-pack3.git=v2.1.1', + 'StackStorm-Exchange/stackstorm-pack4', + 'git://StackStorm-Exchange/stackstorm-pack5=v2.1.1', + 'git://StackStorm-Exchange/stackstorm-pack6.git', + 'git@github.com:foo/pack7.git' + 'git@github.com:foo/pack8.git=v3.2.1', + 'file:///home/vagrant/stackstorm-pack9', + 'file://localhost/home/vagrant/stackstorm-pack10', + 'ssh:///AutomationStackStorm11', + 'ssh://joe@local/AutomationStackStorm12' +] + +DOWNLOADED_OR_INSTALLED_PACK_METAdATA = { + # No dependencies. + "no_dependencies": { + "version": "0.4.0", + "name": "no_dependencies", + "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-no_dependencies", + "author": "st2-dev", + "keywords": ["some", "search", "another", "terms"], + "email": "info@stackstorm.com", + "description": "st2 pack to test package management pipeline", + }, + # One uninstalled and one installed dependency packs. + "test2": { + "version": "0.5.0", + "name": "test2", + "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test2", + "author": "stanley", + "keywords": ["some", "special", "terms"], + "email": "info@stackstorm.com", + "description": "another st2 pack to test package management pipeline", + "dependencies": ['uninstalled_pack', 'no_dependencies'] + }, + # List of uninstalled dependency packs. + "test3": { + "version": "0.6.0", + "stackstorm_version": ">=1.6.0, <2.2.0", + "name": "test3", + "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test3", + "author": "stanley", + "keywords": ["some", "special", "terms"], + "email": "info@stackstorm.com", + "description": "another st2 pack to test package management pipeline", + "dependencies": UNINSTALLED_PACKS + }, + # One conflict pack with existing pack. + "test4": { + "version": "0.7.0", + "stackstorm_version": ">=1.6.0, <2.2.0", + "name": "test4", + "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test4", + "author": "stanley", + "keywords": ["some", "special", "terms"], + "email": "info@stackstorm.com", + "description": "another st2 pack to test package management pipeline", + "dependencies": [ + "test2=v0.4.0" + ] + }, + # One uninstalled conflict pack. + "test5": { + "version": "0.7.0", + "stackstorm_version": ">=1.6.0, <2.2.0", + "name": "test4", + "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test4", + "author": "stanley", + "keywords": ["some", "special", "terms"], "email": "info@stackstorm.com", + "description": "another st2 pack to test package management pipeline", + "dependencies": ["uninstalled_pack=v0.4.0"] + }, + # One dependency pack without version. It is not checked against conflict. + "test6": { + "version": "0.7.0", + "stackstorm_version": ">=1.6.0, <2.2.0", + "name": "test4", + "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test4", + "author": "stanley", + "keywords": ["some", "special", "terms"], "email": "info@stackstorm.com", + "description": "another st2 pack to test package management pipeline", + "dependencies": ["test2"] + } +} + + +def mock_get_dependency_list(pack): + """ + Mock get_dependency_list function which return dependencies list + """ + dependencies = None + + if pack in DOWNLOADED_OR_INSTALLED_PACK_METAdATA: + metadata = DOWNLOADED_OR_INSTALLED_PACK_METAdATA[pack] + dependencies = metadata.get('dependencies', None) + + return dependencies + + +def mock_get_pack_version(pack): + """ + Mock get_pack_version function which return mocked pack version + """ + version = None + + if pack in DOWNLOADED_OR_INSTALLED_PACK_METAdATA: + metadata = DOWNLOADED_OR_INSTALLED_PACK_METAdATA[pack] + version = metadata.get('version', None) + + return version + + +@mock.patch('pack_mgmt.get_pack_dependencies.get_dependency_list', mock_get_dependency_list) +@mock.patch('pack_mgmt.get_pack_dependencies.get_pack_version', mock_get_pack_version) +class GetPackDependenciesTestCase(BaseActionTestCase): + action_cls = GetPackDependencies + + def setUp(self): + super(GetPackDependenciesTestCase, self).setUp() + + def test_run_get_pack_dependencies_with_nested_zero_value(self): + action = self.get_action_instance() + packs_status = {"test": "Success."} + nested = 0 + + result = action.run(packs_status=packs_status, nested=nested) + self.assertEqual(result, {}) + + def test_run_get_pack_dependencies_with_empty_packs_status(self): + action = self.get_action_instance() + packs_status = None + nested = 3 + + result = action.run(packs_status=packs_status, nested=nested) + self.assertEqual(result, {}) + + def test_run_get_pack_dependencies_with_failed_packs_status(self): + action = self.get_action_instance() + packs_status = {"test": "Failed."} + nested = 2 + + result = action.run(packs_status=packs_status, nested=nested) + self.assertEqual(result['dependency_list'], []) + self.assertEqual(result['conflict_list'], []) + self.assertEqual(result['nested'], nested - 1) + + def test_run_get_pack_dependencies_with_failed_and_succeeded_packs_status(self): + action = self.get_action_instance() + packs_status = {"test": "Failed.", "test2": "Success."} + nested = 2 + + result = action.run(packs_status=packs_status, nested=nested) + self.assertEqual(result['dependency_list'], [UNINSTALLED_PACK]) + self.assertEqual(result['conflict_list'], []) + self.assertEqual(result['nested'], nested - 1) + + def test_run_get_pack_dependencies_with_no_dependency(self): + action = self.get_action_instance() + packs_status = {"no_dependencies": "Success."} + nested = 3 + + result = action.run(packs_status=packs_status, nested=nested) + self.assertEqual(result['dependency_list'], []) + self.assertEqual(result['conflict_list'], []) + self.assertEqual(result['nested'], nested - 1) + + def test_run_get_pack_dependencies_with_dependency(self): + action = self.get_action_instance() + packs_status = {"test2": "Success."} + nested = 1 + + result = action.run(packs_status=packs_status, nested=nested) + self.assertEqual(result['dependency_list'], [UNINSTALLED_PACK]) + self.assertEqual(result['conflict_list'], []) + self.assertEqual(result['nested'], nested - 1) + + def test_run_get_pack_dependencies_with_dependencies(self): + action = self.get_action_instance() + packs_status = {"test3": "Success."} + nested = 1 + + result = action.run(packs_status=packs_status, nested=nested) + self.assertEqual(result['dependency_list'], UNINSTALLED_PACKS) + self.assertEqual(result['conflict_list'], []) + self.assertEqual(result['nested'], nested - 1) + + def test_run_get_pack_dependencies_with_existing_pack_conflict(self): + action = self.get_action_instance() + packs_status = {"test2": "Success.", "test4": "Success."} + nested = 1 + + result = action.run(packs_status=packs_status, nested=nested) + self.assertEqual(result['dependency_list'], [UNINSTALLED_PACK]) + self.assertEqual(result['conflict_list'], ['test2=v0.4.0']) + self.assertEqual(result['nested'], nested - 1) + + def test_run_get_pack_dependencies_with_dependency_conflict(self): + action = self.get_action_instance() + packs_status = {"test2": "Success.", "test5": "Success."} + nested = 1 + + result = action.run(packs_status=packs_status, nested=nested) + self.assertEqual(result['dependency_list'], ['uninstalled_pack']) + self.assertEqual(result['conflict_list'], ['uninstalled_pack=v0.4.0']) + self.assertEqual(result['nested'], nested - 1) + + def test_run_get_pack_dependencies_with_no_version(self): + action = self.get_action_instance() + packs_status = {"test2": "Success.", "test6": "Success."} + nested = 1 + + result = action.run(packs_status=packs_status, nested=nested) + self.assertEqual(result['dependency_list'], [UNINSTALLED_PACK]) + self.assertEqual(result['conflict_list'], []) + self.assertEqual(result['nested'], nested - 1) diff --git a/contrib/packs/tests/test_virtualenv_setup_prerun.py b/contrib/packs/tests/test_virtualenv_setup_prerun.py new file mode 100644 index 0000000000..141aab3026 --- /dev/null +++ b/contrib/packs/tests/test_virtualenv_setup_prerun.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from st2tests.base import BaseActionTestCase + +from pack_mgmt.virtualenv_setup_prerun import PacksTransformationAction + + +class VirtualenvSetUpPreRunTestCase(BaseActionTestCase): + action_cls = PacksTransformationAction + + def setUp(self): + super(VirtualenvSetUpPreRunTestCase, self).setUp() + + def test_run_with_pack_list(self): + action = self.get_action_instance() + result = action.run(packs_status={'test1': 'Success.', 'test2': 'Success.'}, + packs_list=['test3', 'test4']) + + self.assertEqual(result, ['test3', 'test4', 'test1', 'test2']) + + def test_run_with_none_pack_list(self): + action = self.get_action_instance() + result = action.run(packs_status={'test1': 'Success.', 'test2': 'Success.'}, + packs_list=None) + + self.assertEqual(result, ['test1', 'test2']) + + def test_run_with_failed_status(self): + action = self.get_action_instance() + result = action.run(packs_status={'test1': 'Failed.', 'test2': 'Success.'}, + packs_list=['test3', 'test4']) + + self.assertEqual(result, ['test3', 'test4', 'test2']) diff --git a/st2api/st2api/controllers/v1/packs.py b/st2api/st2api/controllers/v1/packs.py index 9be03b504d..6478037f55 100644 --- a/st2api/st2api/controllers/v1/packs.py +++ b/st2api/st2api/controllers/v1/packs.py @@ -103,6 +103,9 @@ def post(self, pack_install_request, requester_user=None): if pack_install_request.force: parameters['force'] = True + if pack_install_request.skip_dependencies: + parameters['skip_dependencies'] = True + if not requester_user: requester_user = UserDB(cfg.CONF.system_user.user) diff --git a/st2api/tests/unit/controllers/v1/test_packs.py b/st2api/tests/unit/controllers/v1/test_packs.py index 03d62ed2e7..d6a09f00c1 100644 --- a/st2api/tests/unit/controllers/v1/test_packs.py +++ b/st2api/tests/unit/controllers/v1/test_packs.py @@ -177,6 +177,16 @@ def test_install_with_force_parameter(self, _handle_schedule_execution): self.assertEqual(resp.status_int, 202) self.assertEqual(resp.json, {'execution_id': '123'}) + @mock.patch.object(ActionExecutionsControllerMixin, '_handle_schedule_execution') + def test_install_with_skip_dependencies_parameter(self, _handle_schedule_execution): + _handle_schedule_execution.return_value = Response(json={'id': '123'}) + payload = {'packs': ['some'], 'skip_dependencies': True} + + resp = self.app.post_json('/v1/packs/install', payload) + + self.assertEqual(resp.status_int, 202) + self.assertEqual(resp.json, {'execution_id': '123'}) + @mock.patch.object(ActionExecutionsControllerMixin, '_handle_schedule_execution') def test_uninstall(self, _handle_schedule_execution): _handle_schedule_execution.return_value = Response(json={'id': '123'}) diff --git a/st2client/st2client/commands/pack.py b/st2client/st2client/commands/pack.py index 4e7c7cb084..533d78d78a 100644 --- a/st2client/st2client/commands/pack.py +++ b/st2client/st2client/commands/pack.py @@ -96,7 +96,7 @@ def __init__(self, *args, **kwargs): detail_arg_grp = self.parser.add_mutually_exclusive_group() detail_arg_grp.add_argument('--attr', nargs='+', - default=['name', 'description', 'version', 'author'], + default=['ref', 'name', 'description', 'version', 'author'], help=('List of attributes to include in the ' 'output. "all" or unspecified will ' 'return all attributes.')) @@ -130,7 +130,8 @@ def run_and_print(self, args, **kwargs): if getattr(execution, 'parent', None) == parent_id: status = execution.status - name = execution.context['chain']['name'] + name = execution.context['orquesta']['task_name'] \ + if 'orquesta' in execution.context else execution.context['chain']['name'] if status == LIVEACTION_STATUS_SCHEDULED: indicator.add_stage(status, name) @@ -194,6 +195,10 @@ def __init__(self, resource, *args, **kwargs): action='store_true', default=False, help='Force pack installation.') + self.parser.add_argument('--skip-dependencies', + action='store_true', + default=False, + help='Skip pack dependency installation.') def run(self, args, **kwargs): is_structured_output = args.json or args.yaml @@ -203,7 +208,8 @@ def run(self, args, **kwargs): if not is_structured_output: self._get_content_counts_for_pack(args, **kwargs) - return self.manager.install(args.packs, python3=args.python3, force=args.force, **kwargs) + return self.manager.install(args.packs, python3=args.python3, force=args.force, + skip_dependencies=args.skip_dependencies, **kwargs) def _get_content_counts_for_pack(self, args, **kwargs): # Global content list, excluding "tests" @@ -258,7 +264,7 @@ def _print_pack_content(pack_name, pack_content): def run_and_print(self, args, **kwargs): instance = super(PackInstallCommand, self).run_and_print(args, **kwargs) # Hack to get a list of resolved references of installed packs - packs = instance.result['tasks'][1]['result']['result'] + packs = instance.result['output']['packs_list'] if len(packs) == 1: pack_instance = self.app.client.managers['Pack'].get_by_ref_or_id(packs[0], **kwargs) @@ -270,7 +276,7 @@ def run_and_print(self, args, **kwargs): pack_instances = [] for pack in all_pack_instances: - if pack.name in packs: + if pack.name in packs or pack.ref in packs: pack_instances.append(pack) self.print_output(pack_instances, table.MultiColumnTable, @@ -318,7 +324,7 @@ def run_and_print(self, args, **kwargs): pack_instances = [] for pack in all_pack_instances: - if pack.name in packs: + if pack.name in packs or pack.ref in packs: pack_instances.append(pack) if pack in remaining_pack_instances: raise OperationFailureException('Pack %s has not been removed properly' diff --git a/st2client/st2client/models/core.py b/st2client/st2client/models/core.py index f9d2a86838..627b7183aa 100644 --- a/st2client/st2client/models/core.py +++ b/st2client/st2client/models/core.py @@ -496,12 +496,13 @@ class AsyncRequest(Resource): class PackResourceManager(ResourceManager): @add_auth_token_to_kwargs_from_env - def install(self, packs, force=False, python3=False, **kwargs): + def install(self, packs, force=False, python3=False, skip_dependencies=False, **kwargs): url = '/%s/install' % (self.resource.get_url_path_name()) payload = { 'packs': packs, 'force': force, - 'python3': python3 + 'python3': python3, + 'skip_dependencies': skip_dependencies } response = self.client.post(url, payload, **kwargs) if response.status_code != http_client.OK: diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index 535876b1c6..91e299ff05 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -5028,6 +5028,11 @@ definitions: description: Force pack installation. type: boolean default: false + skip_dependencies: + type: boolean + description: Set to True to skip pack dependency installations. + required: false + default: false PacksUninstall: type: object properties: diff --git a/st2common/st2common/openapi.yaml.j2 b/st2common/st2common/openapi.yaml.j2 index d69e73eed4..253f45cfde 100644 --- a/st2common/st2common/openapi.yaml.j2 +++ b/st2common/st2common/openapi.yaml.j2 @@ -5024,6 +5024,11 @@ definitions: description: Force pack installation. type: boolean default: false + skip_dependencies: + type: boolean + description: Set to True to skip pack dependency installations. + required: false + default: false PacksUninstall: type: object properties: diff --git a/st2common/st2common/util/pack_management.py b/st2common/st2common/util/pack_management.py index ab323ffcb6..b32961b3d5 100644 --- a/st2common/st2common/util/pack_management.py +++ b/st2common/st2common/util/pack_management.py @@ -241,19 +241,27 @@ def clone_repo(temp_dir, repo_url, verify_ssl=True, ref='master'): # We're trying to figure out which branch the ref is actually on, # since there's no direct way to check for this in git-python. branches = repo.git.branch('-a', '--contains', gitref.hexsha) # pylint: disable=no-member - branches = branches.replace('*', '').split() - if active_branch.name not in branches or use_branch: - branch = 'origin/%s' % ref if use_branch else branches[0] - short_branch = ref if use_branch else branches[0].split('/')[-1] - repo.git.checkout('-b', short_branch, branch) - branch = repo.head.reference - else: - branch = repo.active_branch.name + # Git tags aren't necessarily on a branch. + # If this is the case, gitref will be the tag name, but branches will be + # empty. + # We also need to checkout tags slightly differently than branches. + if branches: + branches = branches.replace('*', '').split() + + if active_branch.name not in branches or use_branch: + branch = 'origin/%s' % ref if use_branch else branches[0] + short_branch = ref if use_branch else branches[0].split('/')[-1] + repo.git.checkout('-b', short_branch, branch) + branch = repo.head.reference + else: + branch = repo.active_branch.name - repo.git.checkout(gitref.hexsha) # pylint: disable=no-member - repo.git.branch('-f', branch, gitref.hexsha) # pylint: disable=no-member - repo.git.checkout(branch) + repo.git.checkout(gitref.hexsha) # pylint: disable=no-member + repo.git.branch('-f', branch, gitref.hexsha) # pylint: disable=no-member + repo.git.checkout(branch) + else: + repo.git.checkout('v%s' % ref) # pylint: disable=no-member return temp_dir