From 7dec8a2d3cbe380bf8e82d25d99015b0a36e46e5 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 19 Jul 2018 13:10:09 +0200 Subject: [PATCH 01/24] Move pack download related functionality into new pack_management.download module. This way code is decoupled better and can be re-used in other places. --- .../st2common/pack_management/__init__.py | 0 .../st2common/pack_management/download.py | 374 ++++++++++++++++++ 2 files changed, 374 insertions(+) create mode 100644 st2common/st2common/pack_management/__init__.py create mode 100644 st2common/st2common/pack_management/download.py diff --git a/st2common/st2common/pack_management/__init__.py b/st2common/st2common/pack_management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/st2common/st2common/pack_management/download.py b/st2common/st2common/pack_management/download.py new file mode 100644 index 0000000000..a543e08f73 --- /dev/null +++ b/st2common/st2common/pack_management/download.py @@ -0,0 +1,374 @@ +# -*- coding: utf-8 -*- +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module containing pack download and install related functions. +""" + +from __future__ import absolute_import + +import os +import shutil +import hashlib +import stat +import re + +from git.repo import Repo +from gitdb.exc import BadName, BadObject +from lockfile import LockFile + +from st2common import log as logging +from st2common.content import utils +from st2common.constants.pack import MANIFEST_FILE_NAME +from st2common.constants.pack import PACK_RESERVED_CHARACTERS +from st2common.constants.pack import PACK_VERSION_SEPARATOR +from st2common.constants.pack import PACK_VERSION_REGEX +from st2common.services.packs import get_pack_from_index +from st2common.util.pack import get_pack_metadata +from st2common.util.pack import get_pack_ref_from_metadata +from st2common.util.green import shell +from st2common.util.versioning import complex_semver_match +from st2common.util.versioning import get_stackstorm_version + +__all__ = [ + 'download_pack' +] + +LOG = logging.getLogger(__name__) + +CONFIG_FILE = 'config.yaml' +CURRENT_STACKSTROM_VERSION = get_stackstorm_version() + + +def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, force=False, + proxy_config=None, force_permissions=True, logger=LOG): + """ + Download the pack and move it to /opt/stackstorm/packs. + + :param abs_repo_base: Path where the pack should be installed to. + :type abs_repo_base: ``str`` + + :param pack: Pack name. + :rtype pack: ``str`` + + :param force_permissions: True to force 770 permission on all the pack content. + :type force_permissions: ``bool`` + + :param force: Force the installation and ignore / delete the lock file if it already exists. + :type force: ``bool`` + + :return: (pack_url, pack_ref, result) + :rtype: tuple + """ + proxy_config = proxy_config or {} + + pack_url, pack_version = get_repo_url(pack, proxy_config=proxy_config) + + result = [pack_url, None, None] + + temp_dir_name = hashlib.md5(pack_url).hexdigest() + lock_file = LockFile('/tmp/%s' % (temp_dir_name)) + lock_file_path = lock_file.lock_file + + if force: + logger.debug('Force mode is enabled, deleting lock file...') + + try: + os.unlink(lock_file_path) + except OSError: + # Lock file doesn't exist or similar + pass + + with lock_file: + try: + user_home = os.path.expanduser('~') + abs_local_path = os.path.join(user_home, temp_dir_name) + + # 1. Clone / download the repo + clone_repo(temp_dir=abs_local_path, repo_url=pack_url, verify_ssl=verify_ssl, + ref=pack_version) + + pack_ref = get_pack_ref(pack_dir=abs_local_path) + result[0] = pack_ref + + # 2. Verify that the pack version if compatible with current StackStorm version + if not force: + verify_pack_version(pack_dir=abs_local_path) + + # 3. Move pack to the final location + move_result = move_pack(abs_repo_base=abs_repo_base, pack_name=pack_ref, + abs_local_path=abs_local_path, + force_permissions=force_permissions, + logger=logger) + result[2] = move_result + finally: + cleanup_repo(abs_local_path=abs_local_path) + + return tuple(result) + + +def get_repo_url(pack, proxy_config=None): + """ + Retrieve pack repo url. + + :rtype: ``str`` + + :return: (repo_url, version) + :rtype: tuple + """ + pack_and_version = pack.split(PACK_VERSION_SEPARATOR) + name_or_url = pack_and_version[0] + version = pack_and_version[1] if len(pack_and_version) > 1 else None + + if len(name_or_url.split('/')) == 1: + pack = get_pack_from_index(name_or_url, proxy_config=proxy_config) + + if not pack: + raise Exception('No record of the "%s" pack in the index.' % (name_or_url)) + + return (pack['repo_url'], version) + else: + return (eval_repo_url(name_or_url), version) + + +def eval_repo_url(repo_url): + """ + Allow passing short GitHub style URLs. + """ + if not repo_url: + raise Exception('No valid repo_url provided or could be inferred.') + + if repo_url.startswith("file://"): + return repo_url + else: + if len(repo_url.split('/')) == 2 and 'git@' not in repo_url: + url = 'https://github.com/{}'.format(repo_url) + else: + url = repo_url + return url + + +def clone_repo(temp_dir, repo_url, verify_ssl=True, ref='master'): + # Switch to non-interactive mode + os.environ['GIT_TERMINAL_PROMPT'] = '0' + os.environ['GIT_ASKPASS'] = '/bin/echo' + + # Disable SSL cert checking if explictly asked + if not verify_ssl: + os.environ['GIT_SSL_NO_VERIFY'] = 'true' + + # Clone the repo from git; we don't use shallow copying + # because we want the user to work with the repo in the + # future. + repo = Repo.clone_from(repo_url, temp_dir) + active_branch = repo.active_branch + + use_branch = False + + # Special case when a default repo branch is not "master" + # No ref provided so we just use a default active branch + if (not ref or ref == active_branch.name) and repo.active_branch.object == repo.head.commit: + gitref = repo.active_branch.object + else: + # Try to match the reference to a branch name (i.e. "master") + gitref = get_gitref(repo, 'origin/%s' % ref) + if gitref: + use_branch = True + + # Try to match the reference to a commit hash, a tag, or "master" + if not gitref: + gitref = get_gitref(repo, ref) + + # Try to match the reference to a "vX.Y.Z" tag + if not gitref and re.match(PACK_VERSION_REGEX, ref): + gitref = get_gitref(repo, 'v%s' % ref) + + # Giving up ¯\_(ツ)_/¯ + if not gitref: + format_values = [ref, repo_url] + msg = '"%s" is not a valid version, hash, tag or branch in %s.' + + valid_versions = get_valid_versions_for_repo(repo=repo) + if len(valid_versions) >= 1: + valid_versions_string = ', '.join(valid_versions) + + msg += ' Available versions are: %s.' + format_values.append(valid_versions_string) + + raise ValueError(msg % tuple(format_values)) + + # We're trying to figure out which branch the ref is actually on, + # since there's no direct way to check for this in git-python. + branches = repo.git.branch('-a', '--contains', gitref.hexsha) # pylint: disable=no-member + branches = branches.replace('*', '').split() + + if active_branch.name not in branches or use_branch: + branch = 'origin/%s' % ref if use_branch else branches[0] + short_branch = ref if use_branch else branches[0].split('/')[-1] + repo.git.checkout('-b', short_branch, branch) + branch = repo.head.reference + else: + branch = repo.active_branch.name + + repo.git.checkout(gitref.hexsha) # pylint: disable=no-member + repo.git.branch('-f', branch, gitref.hexsha) # pylint: disable=no-member + repo.git.checkout(branch) + + return temp_dir + + +def move_pack(abs_repo_base, pack_name, abs_local_path, force_permissions=True, logger=LOG): + """ + Move pack directory into the final location. + """ + desired, message = is_desired_pack(abs_local_path, pack_name) + + if desired: + to = abs_repo_base + dest_pack_path = os.path.join(abs_repo_base, pack_name) + if os.path.exists(dest_pack_path): + logger.debug('Removing existing pack %s in %s to replace.', pack_name, + dest_pack_path) + + # Ensure to preserve any existing configuration + old_config_file = os.path.join(dest_pack_path, CONFIG_FILE) + new_config_file = os.path.join(abs_local_path, CONFIG_FILE) + + if os.path.isfile(old_config_file): + shutil.move(old_config_file, new_config_file) + + shutil.rmtree(dest_pack_path) + + logger.debug('Moving pack from %s to %s.', abs_local_path, to) + shutil.move(abs_local_path, dest_pack_path) + + # post move fix all permissions + if force_permissions: + apply_pack_permissions(pack_path=dest_pack_path) + message = 'Success.' + elif message: + message = 'Failure : %s' % message + + return (desired, message) + + +def apply_pack_permissions(pack_path): + """ + Recursively apply permission 770 to pack and its contents. + """ + # 1. switch owner group to configured group + pack_group = utils.get_pack_group() + + if pack_group: + shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path]) + + # 2. Setup the right permissions and group ownership + # These mask is same as mode = 775 + mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH + os.chmod(pack_path, mode) + + # Yuck! Since os.chmod does not support chmod -R walk manually. + for root, dirs, files in os.walk(pack_path): + for d in dirs: + os.chmod(os.path.join(root, d), mode) + for f in files: + os.chmod(os.path.join(root, f), mode) + + +def cleanup_repo(abs_local_path): + # basic lock checking etc? + if os.path.isdir(abs_local_path): + shutil.rmtree(abs_local_path) + + +# Utility functions + +def is_desired_pack(abs_pack_path, pack_name): + # path has to exist. + if not os.path.exists(abs_pack_path): + return (False, 'Pack "%s" not found or it\'s missing a "pack.yaml" file.' % + (pack_name)) + + # should not include reserved characters + for character in PACK_RESERVED_CHARACTERS: + if character in pack_name: + return (False, 'Pack name "%s" contains reserved character "%s"' % + (pack_name, character)) + + # must contain a manifest file. Empty file is ok for now. + if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE_NAME)): + return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE_NAME)) + + return (True, '') + + +def verify_pack_version(pack_dir): + """ + Verify that the pack works with the currently running StackStorm version. + """ + pack_metadata = get_pack_metadata(pack_dir=pack_dir) + pack_name = pack_metadata.get('name', None) + required_stackstorm_version = pack_metadata.get('stackstorm_version', None) + + # If stackstorm_version attribute is speficied, verify that the pack works with currently + # running version of StackStorm + if required_stackstorm_version: + if not complex_semver_match(CURRENT_STACKSTROM_VERSION, required_stackstorm_version): + msg = ('Pack "%s" requires StackStorm "%s", but current version is "%s". ' % + (pack_name, required_stackstorm_version, CURRENT_STACKSTROM_VERSION), + 'You can override this restriction by providing the "force" flag, but ', + 'the pack is not guaranteed to work.') + raise ValueError(msg) + + return True + + +def get_gitref(repo, ref): + """ + Retrieve git repo reference if available. + """ + try: + return repo.commit(ref) + except (BadName, BadObject): + return False + + +def get_valid_versions_for_repo(repo): + """ + Retrieve valid versions (tags) for a particular repo (pack). + + It does so by introspecting available tags. + + :rtype: ``list`` of ``str`` + """ + valid_versions = [] + + for tag in repo.tags: + if tag.name.startswith('v') and re.match(PACK_VERSION_REGEX, tag.name[1:]): + # Note: We strip leading "v" from the version number + valid_versions.append(tag.name[1:]) + + return valid_versions + + +def get_pack_ref(pack_dir): + """ + Read pack reference from the metadata file and sanitize it. + """ + metadata = get_pack_metadata(pack_dir=pack_dir) + pack_ref = get_pack_ref_from_metadata(metadata=metadata, + pack_directory_name=None) + return pack_ref From edacb37f2f3832773d9c1f5d644529207d5ee3c7 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 19 Jul 2018 13:14:58 +0200 Subject: [PATCH 02/24] Update affected packs.download action. --- contrib/packs/actions/pack_mgmt/download.py | 292 +------------------- 1 file changed, 12 insertions(+), 280 deletions(-) diff --git a/contrib/packs/actions/pack_mgmt/download.py b/contrib/packs/actions/pack_mgmt/download.py index 5e6438a294..c2bfce0fad 100644 --- a/contrib/packs/actions/pack_mgmt/download.py +++ b/contrib/packs/actions/pack_mgmt/download.py @@ -15,33 +15,15 @@ # limitations under the License. import os -import shutil -import hashlib -import stat -import re import six -from git.repo import Repo -from gitdb.exc import BadName, BadObject -from lockfile import LockFile from st2common.runners.base_action import Action -from st2common.content import utils -from st2common.constants.pack import MANIFEST_FILE_NAME -from st2common.constants.pack import PACK_RESERVED_CHARACTERS -from st2common.constants.pack import PACK_VERSION_SEPARATOR -from st2common.constants.pack import PACK_VERSION_REGEX -from st2common.services.packs import get_pack_from_index -from st2common.util.pack import get_pack_metadata -from st2common.util.pack import get_pack_ref_from_metadata -from st2common.util.green import shell -from st2common.util.versioning import complex_semver_match -from st2common.util.versioning import get_stackstorm_version +from st2common.pack_management.download import download_pack -CONFIG_FILE = 'config.yaml' - - -CURRENT_STACKSTROM_VERSION = get_stackstorm_version() +__all__ = [ + 'DownloadGitRepoAction' +] class DownloadGitRepoAction(Action): @@ -85,203 +67,21 @@ def run(self, packs, abs_repo_base, verifyssl=True, force=False): result = {} for pack in packs: - pack_url, pack_version = self._get_repo_url(pack, proxy_config=self.proxy_config) - - temp_dir_name = hashlib.md5(pack_url).hexdigest() - lock_file = LockFile('/tmp/%s' % (temp_dir_name)) - lock_file_path = lock_file.lock_file - - if force: - self.logger.debug('Force mode is enabled, deleting lock file...') - - try: - os.unlink(lock_file_path) - except OSError: - # Lock file doesn't exist or similar - pass - - with lock_file: - try: - user_home = os.path.expanduser('~') - abs_local_path = os.path.join(user_home, temp_dir_name) - self._clone_repo(temp_dir=abs_local_path, repo_url=pack_url, - verifyssl=verifyssl, ref=pack_version) - - pack_ref = self._get_pack_ref(abs_local_path) - - # Verify that the pack version if compatible with current StackStorm version - if not force: - self._verify_pack_version(pack_dir=abs_local_path) - - result[pack_ref] = self._move_pack(abs_repo_base, pack_ref, abs_local_path) - finally: - self._cleanup_repo(abs_local_path) + pack_result = download_pack(pack=pack, abs_repo_base=abs_repo_base, + verify_ssl=verifyssl, force=force, + proxy_config=self.proxy_config, + force_permissions=True, + logger=self.logger) + pack_url, pack_ref, pack_result = pack_result + result[pack_ref] = pack_result return self._validate_result(result=result, repo_url=pack_url) - @staticmethod - def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'): - # Switch to non-interactive mode - os.environ['GIT_TERMINAL_PROMPT'] = '0' - os.environ['GIT_ASKPASS'] = '/bin/echo' - - # Disable SSL cert checking if explictly asked - if not verifyssl: - os.environ['GIT_SSL_NO_VERIFY'] = 'true' - - # Clone the repo from git; we don't use shallow copying - # because we want the user to work with the repo in the - # future. - repo = Repo.clone_from(repo_url, temp_dir) - active_branch = repo.active_branch - - use_branch = False - - # Special case when a default repo branch is not "master" - # No ref provided so we just use a default active branch - if (not ref or ref == active_branch.name) and repo.active_branch.object == repo.head.commit: - gitref = repo.active_branch.object - else: - # Try to match the reference to a branch name (i.e. "master") - gitref = DownloadGitRepoAction._get_gitref(repo, 'origin/%s' % ref) - if gitref: - use_branch = True - - # Try to match the reference to a commit hash, a tag, or "master" - if not gitref: - gitref = DownloadGitRepoAction._get_gitref(repo, ref) - - # Try to match the reference to a "vX.Y.Z" tag - if not gitref and re.match(PACK_VERSION_REGEX, ref): - gitref = DownloadGitRepoAction._get_gitref(repo, 'v%s' % ref) - - # Giving up ¯\_(ツ)_/¯ - if not gitref: - format_values = [ref, repo_url] - msg = '"%s" is not a valid version, hash, tag or branch in %s.' - - valid_versions = DownloadGitRepoAction._get_valid_versions_for_repo(repo=repo) - if len(valid_versions) >= 1: - valid_versions_string = ', '.join(valid_versions) - - msg += ' Available versions are: %s.' - format_values.append(valid_versions_string) - - raise ValueError(msg % tuple(format_values)) - - # We're trying to figure out which branch the ref is actually on, - # since there's no direct way to check for this in git-python. - branches = repo.git.branch('-a', '--contains', gitref.hexsha) # pylint: disable=no-member - branches = branches.replace('*', '').split() - - if active_branch.name not in branches or use_branch: - branch = 'origin/%s' % ref if use_branch else branches[0] - short_branch = ref if use_branch else branches[0].split('/')[-1] - repo.git.checkout('-b', short_branch, branch) - branch = repo.head.reference - else: - branch = repo.active_branch.name - - repo.git.checkout(gitref.hexsha) # pylint: disable=no-member - repo.git.branch('-f', branch, gitref.hexsha) # pylint: disable=no-member - repo.git.checkout(branch) - - return temp_dir - - def _move_pack(self, abs_repo_base, pack_name, abs_local_path): - desired, message = DownloadGitRepoAction._is_desired_pack(abs_local_path, pack_name) - if desired: - to = abs_repo_base - dest_pack_path = os.path.join(abs_repo_base, pack_name) - if os.path.exists(dest_pack_path): - self.logger.debug('Removing existing pack %s in %s to replace.', pack_name, - dest_pack_path) - - # Ensure to preserve any existing configuration - old_config_file = os.path.join(dest_pack_path, CONFIG_FILE) - new_config_file = os.path.join(abs_local_path, CONFIG_FILE) - - if os.path.isfile(old_config_file): - shutil.move(old_config_file, new_config_file) - - shutil.rmtree(dest_pack_path) - - self.logger.debug('Moving pack from %s to %s.', abs_local_path, to) - shutil.move(abs_local_path, dest_pack_path) - # post move fix all permissions. - self._apply_pack_permissions(pack_path=dest_pack_path) - message = 'Success.' - elif message: - message = 'Failure : %s' % message - - return (desired, message) - - def _apply_pack_permissions(self, pack_path): - """ - Will recursively apply permission 770 to pack and its contents. - """ - # 1. switch owner group to configured group - pack_group = utils.get_pack_group() - if pack_group: - shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path]) - - # 2. Setup the right permissions and group ownership - # These mask is same as mode = 775 - mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH - os.chmod(pack_path, mode) - - # Yuck! Since os.chmod does not support chmod -R walk manually. - for root, dirs, files in os.walk(pack_path): - for d in dirs: - os.chmod(os.path.join(root, d), mode) - for f in files: - os.chmod(os.path.join(root, f), mode) - - @staticmethod - def _verify_pack_version(pack_dir): - pack_metadata = DownloadGitRepoAction._get_pack_metadata(pack_dir=pack_dir) - pack_name = pack_metadata.get('name', None) - required_stackstorm_version = pack_metadata.get('stackstorm_version', None) - - # If stackstorm_version attribute is speficied, verify that the pack works with currently - # running version of StackStorm - if required_stackstorm_version: - if not complex_semver_match(CURRENT_STACKSTROM_VERSION, required_stackstorm_version): - msg = ('Pack "%s" requires StackStorm "%s", but current version is "%s". ' % - (pack_name, required_stackstorm_version, CURRENT_STACKSTROM_VERSION), - 'You can override this restriction by providing the "force" flag, but ', - 'the pack is not guaranteed to work.') - raise ValueError(msg) - - @staticmethod - def _is_desired_pack(abs_pack_path, pack_name): - # path has to exist. - if not os.path.exists(abs_pack_path): - return (False, 'Pack "%s" not found or it\'s missing a "pack.yaml" file.' % - (pack_name)) - - # should not include reserved characters - for character in PACK_RESERVED_CHARACTERS: - if character in pack_name: - return (False, 'Pack name "%s" contains reserved character "%s"' % - (pack_name, character)) - - # must contain a manifest file. Empty file is ok for now. - if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE_NAME)): - return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE_NAME)) - - return (True, '') - - @staticmethod - def _cleanup_repo(abs_local_path): - # basic lock checking etc? - if os.path.isdir(abs_local_path): - shutil.rmtree(abs_local_path) - @staticmethod def _validate_result(result, repo_url): atleast_one_success = False sanitized_result = {} + for k, v in six.iteritems(result): atleast_one_success |= v[0] sanitized_result[k] = v[1] @@ -299,71 +99,3 @@ def _validate_result(result, repo_url): raise Exception(message) return sanitized_result - - @staticmethod - def _get_repo_url(pack, proxy_config=None): - pack_and_version = pack.split(PACK_VERSION_SEPARATOR) - name_or_url = pack_and_version[0] - version = pack_and_version[1] if len(pack_and_version) > 1 else None - - if len(name_or_url.split('/')) == 1: - pack = get_pack_from_index(name_or_url, proxy_config=proxy_config) - if not pack: - raise Exception('No record of the "%s" pack in the index.' % name_or_url) - return (pack['repo_url'], version) - else: - return (DownloadGitRepoAction._eval_repo_url(name_or_url), version) - - @staticmethod - def _eval_repo_url(repo_url): - """Allow passing short GitHub style URLs""" - if not repo_url: - raise Exception('No valid repo_url provided or could be inferred.') - if repo_url.startswith("file://"): - return repo_url - else: - if len(repo_url.split('/')) == 2 and 'git@' not in repo_url: - url = 'https://github.com/{}'.format(repo_url) - else: - url = repo_url - return url - - @staticmethod - def _get_pack_metadata(pack_dir): - metadata = get_pack_metadata(pack_dir=pack_dir) - return metadata - - @staticmethod - def _get_pack_ref(pack_dir): - """ - Read pack name from the metadata file and sanitize it. - """ - metadata = DownloadGitRepoAction._get_pack_metadata(pack_dir=pack_dir) - pack_ref = get_pack_ref_from_metadata(metadata=metadata, - pack_directory_name=None) - return pack_ref - - @staticmethod - def _get_valid_versions_for_repo(repo): - """ - Method which returns a valid versions for a particular repo (pack). - - It does so by introspecting available tags. - - :rtype: ``list`` of ``str`` - """ - valid_versions = [] - - for tag in repo.tags: - if tag.name.startswith('v') and re.match(PACK_VERSION_REGEX, tag.name[1:]): - # Note: We strip leading "v" from the version number - valid_versions.append(tag.name[1:]) - - return valid_versions - - @staticmethod - def _get_gitref(repo, ref): - try: - return repo.commit(ref) - except (BadName, BadObject): - return False From 142d54261add6e69916ae71883d03b7247a0a0bc Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 19 Jul 2018 13:16:13 +0200 Subject: [PATCH 03/24] Fix typo. --- st2common/st2common/pack_management/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/pack_management/download.py b/st2common/st2common/pack_management/download.py index a543e08f73..ba05a931d6 100644 --- a/st2common/st2common/pack_management/download.py +++ b/st2common/st2common/pack_management/download.py @@ -102,7 +102,7 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, ref=pack_version) pack_ref = get_pack_ref(pack_dir=abs_local_path) - result[0] = pack_ref + result[1] = pack_ref # 2. Verify that the pack version if compatible with current StackStorm version if not force: From 10d737a5fa95f3148d8410e9cad65546358d485c Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 19 Jul 2018 13:17:32 +0200 Subject: [PATCH 04/24] Use consistent function order. --- .../st2common/pack_management/download.py | 81 +++++++++---------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/st2common/st2common/pack_management/download.py b/st2common/st2common/pack_management/download.py index ba05a931d6..ee8e121ce2 100644 --- a/st2common/st2common/pack_management/download.py +++ b/st2common/st2common/pack_management/download.py @@ -120,47 +120,6 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, return tuple(result) -def get_repo_url(pack, proxy_config=None): - """ - Retrieve pack repo url. - - :rtype: ``str`` - - :return: (repo_url, version) - :rtype: tuple - """ - pack_and_version = pack.split(PACK_VERSION_SEPARATOR) - name_or_url = pack_and_version[0] - version = pack_and_version[1] if len(pack_and_version) > 1 else None - - if len(name_or_url.split('/')) == 1: - pack = get_pack_from_index(name_or_url, proxy_config=proxy_config) - - if not pack: - raise Exception('No record of the "%s" pack in the index.' % (name_or_url)) - - return (pack['repo_url'], version) - else: - return (eval_repo_url(name_or_url), version) - - -def eval_repo_url(repo_url): - """ - Allow passing short GitHub style URLs. - """ - if not repo_url: - raise Exception('No valid repo_url provided or could be inferred.') - - if repo_url.startswith("file://"): - return repo_url - else: - if len(repo_url.split('/')) == 2 and 'git@' not in repo_url: - url = 'https://github.com/{}'.format(repo_url) - else: - url = repo_url - return url - - def clone_repo(temp_dir, repo_url, verify_ssl=True, ref='master'): # Switch to non-interactive mode os.environ['GIT_TERMINAL_PROMPT'] = '0' @@ -295,6 +254,46 @@ def cleanup_repo(abs_local_path): # Utility functions +def get_repo_url(pack, proxy_config=None): + """ + Retrieve pack repo url. + + :rtype: ``str`` + + :return: (repo_url, version) + :rtype: tuple + """ + pack_and_version = pack.split(PACK_VERSION_SEPARATOR) + name_or_url = pack_and_version[0] + version = pack_and_version[1] if len(pack_and_version) > 1 else None + + if len(name_or_url.split('/')) == 1: + pack = get_pack_from_index(name_or_url, proxy_config=proxy_config) + + if not pack: + raise Exception('No record of the "%s" pack in the index.' % (name_or_url)) + + return (pack['repo_url'], version) + else: + return (eval_repo_url(name_or_url), version) + + +def eval_repo_url(repo_url): + """ + Allow passing short GitHub style URLs. + """ + if not repo_url: + raise Exception('No valid repo_url provided or could be inferred.') + + if repo_url.startswith("file://"): + return repo_url + else: + if len(repo_url.split('/')) == 2 and 'git@' not in repo_url: + url = 'https://github.com/{}'.format(repo_url) + else: + url = repo_url + return url + def is_desired_pack(abs_pack_path, pack_name): # path has to exist. From 2cf16b207772c80680c350ded8302ff30b8e1160 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 19 Jul 2018 13:20:34 +0200 Subject: [PATCH 05/24] Update download function to correctly handle the error scenario when we are unable to find pack in the index. --- st2common/st2common/pack_management/download.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/st2common/st2common/pack_management/download.py b/st2common/st2common/pack_management/download.py index ee8e121ce2..956ecd680c 100644 --- a/st2common/st2common/pack_management/download.py +++ b/st2common/st2common/pack_management/download.py @@ -75,7 +75,12 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, """ proxy_config = proxy_config or {} - pack_url, pack_version = get_repo_url(pack, proxy_config=proxy_config) + try: + pack_url, pack_version = get_repo_url(pack, proxy_config=proxy_config) + except Exception as e: + # Pack not found or similar + result = [None, pack, (False, str(e))] + return result result = [pack_url, None, None] From b569c22b09824689757f6f1245b36728d7385f08 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 19 Jul 2018 13:24:21 +0200 Subject: [PATCH 06/24] Update affected tests. --- contrib/packs/tests/test_action_download.py | 35 ++++++++++--------- .../st2common/pack_management/download.py | 5 ++- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/contrib/packs/tests/test_action_download.py b/contrib/packs/tests/test_action_download.py index c951089c09..69c5c35062 100644 --- a/contrib/packs/tests/test_action_download.py +++ b/contrib/packs/tests/test_action_download.py @@ -31,7 +31,8 @@ from st2common.services import packs as pack_service from st2tests.base import BaseActionTestCase -import pack_mgmt.download +import st2common.pack_management.download +from st2common.pack_management.download import eval_repo_url from pack_mgmt.download import DownloadGitRepoAction PACK_INDEX = { @@ -218,7 +219,7 @@ def side_effect(ref): self.assertEqual(result, {'test': 'Success.'}) - @mock.patch.object(DownloadGitRepoAction, '_get_valid_versions_for_repo', + @mock.patch.object(st2common.pack_management.download, 'get_valid_versions_for_repo', mock.Mock(return_value=['1.0.0', '2.0.0'])) def test_run_pack_download_invalid_version(self): self.repo_instance.commit.side_effect = lambda ref: None @@ -234,69 +235,69 @@ def test_download_pack_stackstorm_version_identifier_check(self): action = self.get_action_instance() # Version is satisfied - pack_mgmt.download.CURRENT_STACKSTROM_VERSION = '2.0.0' + st2common.pack_management.download.CURRENT_STACKSTROM_VERSION = '2.0.0' result = action.run(packs=['test3'], abs_repo_base=self.repo_base) self.assertEqual(result['test3'], 'Success.') # Pack requires a version which is not satisfied by current StackStorm version - pack_mgmt.download.CURRENT_STACKSTROM_VERSION = '2.2.0' + st2common.pack_management.download.CURRENT_STACKSTROM_VERSION = '2.2.0' expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' 'current version is "2.2.0"') self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], abs_repo_base=self.repo_base) - pack_mgmt.download.CURRENT_STACKSTROM_VERSION = '2.3.0' + st2common.pack_management.download.CURRENT_STACKSTROM_VERSION = '2.3.0' expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' 'current version is "2.3.0"') self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], abs_repo_base=self.repo_base) - pack_mgmt.download.CURRENT_STACKSTROM_VERSION = '1.5.9' + st2common.pack_management.download.CURRENT_STACKSTROM_VERSION = '1.5.9' expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' 'current version is "1.5.9"') self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], abs_repo_base=self.repo_base) - pack_mgmt.download.CURRENT_STACKSTROM_VERSION = '1.5.0' + st2common.pack_management.download.CURRENT_STACKSTROM_VERSION = '1.5.0' expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' 'current version is "1.5.0"') self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], abs_repo_base=self.repo_base) # Version is not met, but force=true parameter is provided - pack_mgmt.download.CURRENT_STACKSTROM_VERSION = '1.5.0' + st2common.pack_management.download.CURRENT_STACKSTROM_VERSION = '1.5.0' result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=True) self.assertEqual(result['test3'], 'Success.') def test_resolve_urls(self): - url = DownloadGitRepoAction._eval_repo_url( + url = eval_repo_url( "https://github.com/StackStorm-Exchange/stackstorm-test") self.assertEqual(url, "https://github.com/StackStorm-Exchange/stackstorm-test") - url = DownloadGitRepoAction._eval_repo_url( + url = eval_repo_url( "https://github.com/StackStorm-Exchange/stackstorm-test.git") self.assertEqual(url, "https://github.com/StackStorm-Exchange/stackstorm-test.git") - url = DownloadGitRepoAction._eval_repo_url("StackStorm-Exchange/stackstorm-test") + url = eval_repo_url("StackStorm-Exchange/stackstorm-test") self.assertEqual(url, "https://github.com/StackStorm-Exchange/stackstorm-test") - url = DownloadGitRepoAction._eval_repo_url("git://StackStorm-Exchange/stackstorm-test") + url = eval_repo_url("git://StackStorm-Exchange/stackstorm-test") self.assertEqual(url, "git://StackStorm-Exchange/stackstorm-test") - url = DownloadGitRepoAction._eval_repo_url("git://StackStorm-Exchange/stackstorm-test.git") + url = eval_repo_url("git://StackStorm-Exchange/stackstorm-test.git") self.assertEqual(url, "git://StackStorm-Exchange/stackstorm-test.git") - url = DownloadGitRepoAction._eval_repo_url("git@github.com:foo/bar.git") + url = eval_repo_url("git@github.com:foo/bar.git") self.assertEqual(url, "git@github.com:foo/bar.git") - url = DownloadGitRepoAction._eval_repo_url("file:///home/vagrant/stackstorm-test") + url = eval_repo_url("file:///home/vagrant/stackstorm-test") self.assertEqual(url, "file:///home/vagrant/stackstorm-test") - url = DownloadGitRepoAction._eval_repo_url('ssh:///AutomationStackStorm') + url = eval_repo_url('ssh:///AutomationStackStorm') self.assertEqual(url, 'ssh:///AutomationStackStorm') - url = DownloadGitRepoAction._eval_repo_url('ssh://joe@local/AutomationStackStorm') + url = eval_repo_url('ssh://joe@local/AutomationStackStorm') self.assertEqual(url, 'ssh://joe@local/AutomationStackStorm') def test_run_pack_download_edge_cases(self): diff --git a/st2common/st2common/pack_management/download.py b/st2common/st2common/pack_management/download.py index 956ecd680c..319eec5802 100644 --- a/st2common/st2common/pack_management/download.py +++ b/st2common/st2common/pack_management/download.py @@ -44,7 +44,10 @@ from st2common.util.versioning import get_stackstorm_version __all__ = [ - 'download_pack' + 'download_pack', + + 'get_repo_url', + 'eval_repo_url' ] LOG = logging.getLogger(__name__) From 1262b60ec5fa567f870492ab66cd5a5ed26cecf4 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 20 Jul 2018 11:17:50 +0200 Subject: [PATCH 07/24] Add a standalone script for installing a pack and setting a pack virtual environment. This script doesn't depend on RabbitMQ and MongoDB connection and is to be used and Docker and similar environments where content (packs) are pre-baked into container / vm image. --- st2common/bin/st2-install-pack | 42 ++++++++ st2common/st2common/cmd/install_pack.py | 122 ++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100755 st2common/bin/st2-install-pack create mode 100644 st2common/st2common/cmd/install_pack.py diff --git a/st2common/bin/st2-install-pack b/st2common/bin/st2-install-pack new file mode 100755 index 0000000000..cb43364113 --- /dev/null +++ b/st2common/bin/st2-install-pack @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +StackStorm command for installing a pack and setting up pack virtual environment on the local +system where the command runs. + +This command is meant to be used in Docker and similar environments where container / server +where the pack is installed doesn't have access to the database (MongoDB) and message bus +(RabbitMQ). + +Keep in mind that pack still eventually needs to be registered in the database by running +"st2ctl reload --register-all" command on a server with access to the database and message bus. + +NOTE: Ideally for this command to work, whole StackStorm package would be installed on the +system, but at the very least, the following things need to be present: + + * Python 2.7 / 3.x with a recent virtualenv binary + * st2common StackStorm Python package + * /etc/st2/st2.conf config file (user can specify custom pack exchange index, settings there, + etc.) +""" + +import sys + +from st2common.cmd import install_pack + +if __name__ == '__main__': + sys.exit(install_pack.main(sys.argv[1:])) diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py new file mode 100644 index 0000000000..c39784dfa5 --- /dev/null +++ b/st2common/st2common/cmd/install_pack.py @@ -0,0 +1,122 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +from oslo_config import cfg + +from st2common import config +from st2common import log as logging +from st2common.script_setup import setup as common_setup +from st2common.pack_management.download import download_pack +from st2common.util.virtualenvs import setup_pack_virtualenv + +__all__ = [ + 'main' +] + +LOG = logging.getLogger(__name__) + + +def _do_register_cli_opts(opts, ignore_errors=False): + for opt in opts: + try: + cfg.CONF.register_cli_opt(opt) + except: + if not ignore_errors: + raise + + +def _register_cli_opts(): + cli_opts = [ + cfg.StrOpt('pack', default=None, required=True, + help='Name of the pack to install.'), + cfg.BoolOpt('verify-ssl', default=True, + help=('Verify SSL certificate of the Git repo from which the pack is ' + 'downloaded.')), + cfg.BoolOpt('force', default=False, + help='True to force pack installation and ignore install ' + 'lock file if it exists.'), + ] + _do_register_cli_opts(cli_opts) + + +def get_and_set_proxy_config(): + https_proxy = os.environ.get('https_proxy', None) + http_proxy = os.environ.get('http_proxy', None) + proxy_ca_bundle_path = os.environ.get('proxy_ca_bundle_path', None) + no_proxy = os.environ.get('no_proxy', None) + + proxy_config = {} + + if http_proxy or https_proxy: + LOG.debug('Using proxy %s', http_proxy if http_proxy else https_proxy) + + proxy_config = { + 'https_proxy': https_proxy, + 'http_proxy': http_proxy, + 'proxy_ca_bundle_path': proxy_ca_bundle_path, + 'no_proxy': no_proxy + } + + if https_proxy and not os.environ.get('https_proxy', None): + os.environ['https_proxy'] = https_proxy + + if http_proxy and not os.environ.get('http_proxy', None): + os.environ['http_proxy'] = http_proxy + + if no_proxy and not os.environ.get('no_proxy', None): + os.environ['no_proxy'] = no_proxy + + if proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None): + os.environ['no_proxy'] = no_proxy + + return proxy_config + + +def main(argv): + _register_cli_opts() + + # Parse CLI args, set up logging + common_setup(config=config, setup_db=False, register_mq_exchanges=False, + register_internal_trigger_types=False) + + pack = cfg.CONF.pack + verify_ssl = cfg.CONF.verify_ssl + force = cfg.CONF.force + + proxy_config = get_and_set_proxy_config() + + # 1. Download the pac + LOG.info('Installing pack "%s"' % (pack)) + result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, + proxy_config=proxy_config, force_permissions=True) + + success = result[2][0] + + if success: + LOG.info('Successfuly installed pack "%s"' % (pack)) + else: + error = result[2][1] + LOG.error('Failed to install pack "%s": %s' % (pack, error)) + sys.exit(2) + + # 2. Setup pack virtual environment + LOG.info('Setting up virtualenv for pack "%s"' % (pack)) + setup_pack_virtualenv(pack_name=pack, update=False, logger=LOG, + proxy_config=proxy_config, use_python3=False, + no_download=True) + LOG.info('Successfuly set up virtualenv for pack "%s"' % (pack)) From bace62cc2fceca304d10de218388f39c2ee8a301 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 20 Jul 2018 12:46:47 +0200 Subject: [PATCH 08/24] No need for the whole package - move pack management related utility functions in st2common.util.pack_management module. --- contrib/packs/actions/pack_mgmt/download.py | 2 +- contrib/packs/tests/test_action_download.py | 19 ++++++++++--------- st2common/st2common/cmd/install_pack.py | 2 +- .../st2common/pack_management/__init__.py | 0 .../download.py => util/pack_management.py} | 2 +- 5 files changed, 13 insertions(+), 12 deletions(-) delete mode 100644 st2common/st2common/pack_management/__init__.py rename st2common/st2common/{pack_management/download.py => util/pack_management.py} (99%) diff --git a/contrib/packs/actions/pack_mgmt/download.py b/contrib/packs/actions/pack_mgmt/download.py index c2bfce0fad..8869fca635 100644 --- a/contrib/packs/actions/pack_mgmt/download.py +++ b/contrib/packs/actions/pack_mgmt/download.py @@ -19,7 +19,7 @@ import six from st2common.runners.base_action import Action -from st2common.pack_management.download import download_pack +from st2common.util.pack_management import download_pack __all__ = [ 'DownloadGitRepoAction' diff --git a/contrib/packs/tests/test_action_download.py b/contrib/packs/tests/test_action_download.py index 69c5c35062..f6b18332fb 100644 --- a/contrib/packs/tests/test_action_download.py +++ b/contrib/packs/tests/test_action_download.py @@ -31,8 +31,9 @@ from st2common.services import packs as pack_service from st2tests.base import BaseActionTestCase -import st2common.pack_management.download -from st2common.pack_management.download import eval_repo_url +import st2common.util.pack_management +from st2common.util.pack_management import eval_repo_url + from pack_mgmt.download import DownloadGitRepoAction PACK_INDEX = { @@ -219,7 +220,7 @@ def side_effect(ref): self.assertEqual(result, {'test': 'Success.'}) - @mock.patch.object(st2common.pack_management.download, 'get_valid_versions_for_repo', + @mock.patch.object(st2common.util.pack_management, 'get_valid_versions_for_repo', mock.Mock(return_value=['1.0.0', '2.0.0'])) def test_run_pack_download_invalid_version(self): self.repo_instance.commit.side_effect = lambda ref: None @@ -235,38 +236,38 @@ def test_download_pack_stackstorm_version_identifier_check(self): action = self.get_action_instance() # Version is satisfied - st2common.pack_management.download.CURRENT_STACKSTROM_VERSION = '2.0.0' + st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '2.0.0' result = action.run(packs=['test3'], abs_repo_base=self.repo_base) self.assertEqual(result['test3'], 'Success.') # Pack requires a version which is not satisfied by current StackStorm version - st2common.pack_management.download.CURRENT_STACKSTROM_VERSION = '2.2.0' + st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '2.2.0' expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' 'current version is "2.2.0"') self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], abs_repo_base=self.repo_base) - st2common.pack_management.download.CURRENT_STACKSTROM_VERSION = '2.3.0' + st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '2.3.0' expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' 'current version is "2.3.0"') self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], abs_repo_base=self.repo_base) - st2common.pack_management.download.CURRENT_STACKSTROM_VERSION = '1.5.9' + st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '1.5.9' expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' 'current version is "1.5.9"') self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], abs_repo_base=self.repo_base) - st2common.pack_management.download.CURRENT_STACKSTROM_VERSION = '1.5.0' + st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '1.5.0' expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' 'current version is "1.5.0"') self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], abs_repo_base=self.repo_base) # Version is not met, but force=true parameter is provided - st2common.pack_management.download.CURRENT_STACKSTROM_VERSION = '1.5.0' + st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '1.5.0' result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=True) self.assertEqual(result['test3'], 'Success.') diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py index c39784dfa5..23f1a03955 100644 --- a/st2common/st2common/cmd/install_pack.py +++ b/st2common/st2common/cmd/install_pack.py @@ -21,7 +21,7 @@ from st2common import config from st2common import log as logging from st2common.script_setup import setup as common_setup -from st2common.pack_management.download import download_pack +from st2common.util.pack_management import download_pack from st2common.util.virtualenvs import setup_pack_virtualenv __all__ = [ diff --git a/st2common/st2common/pack_management/__init__.py b/st2common/st2common/pack_management/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2common/st2common/pack_management/download.py b/st2common/st2common/util/pack_management.py similarity index 99% rename from st2common/st2common/pack_management/download.py rename to st2common/st2common/util/pack_management.py index 319eec5802..35cc77d219 100644 --- a/st2common/st2common/pack_management/download.py +++ b/st2common/st2common/util/pack_management.py @@ -15,7 +15,7 @@ # limitations under the License. """ -Module containing pack download and install related functions. +Module containing pack management related functions. """ from __future__ import absolute_import From a7649858d6056042f61fe67f4be731541ae75d81 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 20 Jul 2018 12:53:45 +0200 Subject: [PATCH 09/24] st2-install-pack -> st2-pack-install. --- st2common/bin/{st2-install-pack => st2-pack-install} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename st2common/bin/{st2-install-pack => st2-pack-install} (100%) diff --git a/st2common/bin/st2-install-pack b/st2common/bin/st2-pack-install similarity index 100% rename from st2common/bin/st2-install-pack rename to st2common/bin/st2-pack-install From 620f7d8bc6975ff96a98da23b26748a3352024d1 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 20 Jul 2018 16:33:46 +0200 Subject: [PATCH 10/24] Update affected tests. --- st2common/tests/unit/test_pack_management.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/st2common/tests/unit/test_pack_management.py b/st2common/tests/unit/test_pack_management.py index 0f15b49a60..b7be63f670 100644 --- a/st2common/tests/unit/test_pack_management.py +++ b/st2common/tests/unit/test_pack_management.py @@ -29,7 +29,7 @@ from st2common.util.monkey_patch import use_select_poll_workaround use_select_poll_workaround() -from pack_mgmt.download import DownloadGitRepoAction +from st2common.util.pack_management import eval_repo_url __all__ = [ 'InstallPackTestCase' @@ -39,16 +39,16 @@ class InstallPackTestCase(unittest2.TestCase): def test_eval_repo(self): - result = DownloadGitRepoAction._eval_repo_url('stackstorm/st2contrib') + result = eval_repo_url('stackstorm/st2contrib') self.assertEqual(result, 'https://github.com/stackstorm/st2contrib') - result = DownloadGitRepoAction._eval_repo_url('git@github.com:StackStorm/st2contrib.git') + result = eval_repo_url('git@github.com:StackStorm/st2contrib.git') self.assertEqual(result, 'git@github.com:StackStorm/st2contrib.git') repo_url = 'https://github.com/StackStorm/st2contrib.git' - result = DownloadGitRepoAction._eval_repo_url(repo_url) + result = eval_repo_url(repo_url) self.assertEqual(result, repo_url) repo_url = 'https://git-wip-us.apache.org/repos/asf/libcloud.git' - result = DownloadGitRepoAction._eval_repo_url(repo_url) + result = eval_repo_url(repo_url) self.assertEqual(result, repo_url) From b38222877b5fadca5921f85df9a0e555f4550101 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 20 Jul 2018 16:33:52 +0200 Subject: [PATCH 11/24] Add changelog entry. --- CHANGELOG.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 37a4c22392..4217328fcf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,18 @@ Added * Add new ``?tags``, query param filter to the ``/v1/actions`` API endpoint. This query parameter allows users to filter out actions based on the tag name . By default, when no filter values are provided, all actions are returned. (new feature) #4219 +* Add a new standalone standalone ``st2-pack-install`` CLI command. This command installs a pack + (and sets up the pack virtual environment) on the server where it runs. It doesn't register the + content. It only depends on the Python, git and pip binary and ``st2common`` Python package to be + installed on the system where it runs. It doesn't depend on the database (MongoDB) and message + bus (RabbitMQ). + + It's primary meant to be used in scenarios where the content (packs) are baked into the base + container / VM image which is deployed to the cluster. + + Keep in mind that the content itself still needs to be registered with StackStorm at some later + point when access to RabbitMQ and MongoDB is available by running + ``st2ctl reload --register-all``. (new feature) #3912 #4256 Changed ~~~~~~~ @@ -59,7 +71,7 @@ Changed * Migrated runners to using the ``in-requirements.txt`` pattern for "components" in the build system, so the ``Makefile`` correctly generates and installs runner dependencies during testing and packaging. (improvement) (bugfix) #4169 - + Contributed by Nick Maludy (Encore Technologies). * Update ``st2`` CLI to use a more sensible default terminal size for table formatting purposes if we are unable to retrieve terminal size using various system-specific approaches. From 1065abb53a916cc30c45d29eb00f75a782f39d41 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 20 Jul 2018 16:40:59 +0200 Subject: [PATCH 12/24] Add support for installing multiple packs at once to "st2-pack-install" CLI script. --- st2common/st2common/cmd/install_pack.py | 49 +++++++++++++------------ 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py index 23f1a03955..89cf9c61c0 100644 --- a/st2common/st2common/cmd/install_pack.py +++ b/st2common/st2common/cmd/install_pack.py @@ -42,8 +42,8 @@ def _do_register_cli_opts(opts, ignore_errors=False): def _register_cli_opts(): cli_opts = [ - cfg.StrOpt('pack', default=None, required=True, - help='Name of the pack to install.'), + cfg.MultiStrOpt('pack', default=None, required=True, positional=True, + help='Name of the pack to install.'), cfg.BoolOpt('verify-ssl', default=True, help=('Verify SSL certificate of the Git repo from which the pack is ' 'downloaded.')), @@ -94,29 +94,32 @@ def main(argv): common_setup(config=config, setup_db=False, register_mq_exchanges=False, register_internal_trigger_types=False) - pack = cfg.CONF.pack + packs = cfg.CONF.pack verify_ssl = cfg.CONF.verify_ssl force = cfg.CONF.force proxy_config = get_and_set_proxy_config() - # 1. Download the pac - LOG.info('Installing pack "%s"' % (pack)) - result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, - proxy_config=proxy_config, force_permissions=True) - - success = result[2][0] - - if success: - LOG.info('Successfuly installed pack "%s"' % (pack)) - else: - error = result[2][1] - LOG.error('Failed to install pack "%s": %s' % (pack, error)) - sys.exit(2) - - # 2. Setup pack virtual environment - LOG.info('Setting up virtualenv for pack "%s"' % (pack)) - setup_pack_virtualenv(pack_name=pack, update=False, logger=LOG, - proxy_config=proxy_config, use_python3=False, - no_download=True) - LOG.info('Successfuly set up virtualenv for pack "%s"' % (pack)) + for pack in packs: + # 1. Download the pack + LOG.info('Installing pack "%s"' % (pack)) + result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, + proxy_config=proxy_config, force_permissions=True) + + success = result[2][0] + + if success: + LOG.info('Successfuly installed pack "%s"' % (pack)) + else: + error = result[2][1] + LOG.error('Failed to install pack "%s": %s' % (pack, error)) + sys.exit(2) + + # 2. Setup pack virtual environment + LOG.info('Setting up virtualenv for pack "%s"' % (pack)) + setup_pack_virtualenv(pack_name=pack, update=False, logger=LOG, + proxy_config=proxy_config, use_python3=False, + no_download=True) + LOG.info('Successfuly set up virtualenv for pack "%s"' % (pack)) + + return 0 From a090edda70d370e53611e081ff68b110c3f8a555 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 23 Jul 2018 10:35:16 +0200 Subject: [PATCH 13/24] Add missing setup.py entry. --- st2common/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/st2common/setup.py b/st2common/setup.py index a00dc2c0cf..283fa5ddb9 100644 --- a/st2common/setup.py +++ b/st2common/setup.py @@ -61,7 +61,8 @@ 'bin/st2-self-check', 'bin/st2-track-result', 'bin/st2-validate-pack-config', - 'bin/st2-check-license' + 'bin/st2-check-license', + 'bin/st2-pack-install' ], entry_points={ 'st2common.metrics.driver': [ From ace321d2823b087ba77bbde2f963e6f02ecff66d Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 30 Jul 2018 10:58:23 +0200 Subject: [PATCH 14/24] Fix "st2-pack-install" script so the setup virtualenv step also works correctly if user passes in a version (e.g. st2-pack-instal bitcon=0.1.0). --- st2common/st2common/cmd/install_pack.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py index 89cf9c61c0..2371fcc5eb 100644 --- a/st2common/st2common/cmd/install_pack.py +++ b/st2common/st2common/cmd/install_pack.py @@ -106,6 +106,8 @@ def main(argv): result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, proxy_config=proxy_config, force_permissions=True) + # Raw pack name excluding the version + pack_name = result[1] success = result[2][0] if success: @@ -116,8 +118,8 @@ def main(argv): sys.exit(2) # 2. Setup pack virtual environment - LOG.info('Setting up virtualenv for pack "%s"' % (pack)) - setup_pack_virtualenv(pack_name=pack, update=False, logger=LOG, + LOG.info('Setting up virtualenv for pack "%s"' % (pack_name)) + setup_pack_virtualenv(pack_name=pack_name, update=False, logger=LOG, proxy_config=proxy_config, use_python3=False, no_download=True) LOG.info('Successfuly set up virtualenv for pack "%s"' % (pack)) From f38e57bf526c5cb82f39f2f234119dc6fa7d003c Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 30 Jul 2018 11:06:25 +0200 Subject: [PATCH 15/24] Use cleaned pack name everywhere. --- st2common/st2common/cmd/install_pack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py index 2371fcc5eb..d69dd9f966 100644 --- a/st2common/st2common/cmd/install_pack.py +++ b/st2common/st2common/cmd/install_pack.py @@ -111,10 +111,10 @@ def main(argv): success = result[2][0] if success: - LOG.info('Successfuly installed pack "%s"' % (pack)) + LOG.info('Successfuly installed pack "%s"' % (pack_name)) else: error = result[2][1] - LOG.error('Failed to install pack "%s": %s' % (pack, error)) + LOG.error('Failed to install pack "%s": %s' % (pack_name, error)) sys.exit(2) # 2. Setup pack virtual environment @@ -122,6 +122,6 @@ def main(argv): setup_pack_virtualenv(pack_name=pack_name, update=False, logger=LOG, proxy_config=proxy_config, use_python3=False, no_download=True) - LOG.info('Successfuly set up virtualenv for pack "%s"' % (pack)) + LOG.info('Successfuly set up virtualenv for pack "%s"' % (pack_name)) return 0 From 62671af911e79bf1dbad30f397eb10f874a7ea0f Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 30 Jul 2018 11:12:11 +0200 Subject: [PATCH 16/24] Fix typos. --- contrib/packs/actions/pack_mgmt/setup_virtualenv.py | 2 +- scripts/travis/install-and-run-mongodb.sh | 2 +- st2common/st2common/cmd/install_pack.py | 4 ++-- st2common/st2common/cmd/validate_config.py | 2 +- st2common/st2common/models/db/__init__.py | 2 +- st2reactor/st2reactor/container/process_container.py | 7 ++++--- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py index 1d45d044d1..b645ede90c 100644 --- a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py +++ b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py @@ -88,6 +88,6 @@ def run(self, packs, update=False, python3=False, no_download=True): proxy_config=self.proxy_config, use_python3=python3, no_download=no_download) - message = ('Successfuly set up virtualenv for the following packs: %s' % + message = ('Successfully set up virtualenv for the following packs: %s' % (', '.join(packs))) return message diff --git a/scripts/travis/install-and-run-mongodb.sh b/scripts/travis/install-and-run-mongodb.sh index fbe8abdc6f..3c2ae74934 100755 --- a/scripts/travis/install-and-run-mongodb.sh +++ b/scripts/travis/install-and-run-mongodb.sh @@ -46,7 +46,7 @@ MONGODB_PID=$! sleep 5 if ps -p ${MONGODB_PID} > /dev/null; then - echo "MongoDB successfuly started" + echo "MongoDB successfully started" tail -30 /tmp/mongodb.log exit 0 else diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py index d69dd9f966..bcaf1135bc 100644 --- a/st2common/st2common/cmd/install_pack.py +++ b/st2common/st2common/cmd/install_pack.py @@ -111,7 +111,7 @@ def main(argv): success = result[2][0] if success: - LOG.info('Successfuly installed pack "%s"' % (pack_name)) + LOG.info('Successfully installed pack "%s"' % (pack_name)) else: error = result[2][1] LOG.error('Failed to install pack "%s": %s' % (pack_name, error)) @@ -122,6 +122,6 @@ def main(argv): setup_pack_virtualenv(pack_name=pack_name, update=False, logger=LOG, proxy_config=proxy_config, use_python3=False, no_download=True) - LOG.info('Successfuly set up virtualenv for pack "%s"' % (pack_name)) + LOG.info('Successfully set up virtualenv for pack "%s"' % (pack_name)) return 0 diff --git a/st2common/st2common/cmd/validate_config.py b/st2common/st2common/cmd/validate_config.py index 93aea16107..d23e6d5ef7 100644 --- a/st2common/st2common/cmd/validate_config.py +++ b/st2common/st2common/cmd/validate_config.py @@ -76,5 +76,5 @@ def main(): print('Failed to validate pack config.\n%s' % str(e)) return FAILURE_EXIT_CODE - print('Config "%s" successfuly validated against schema in %s.' % (config_path, schema_path)) + print('Config "%s" successfully validated against schema in %s.' % (config_path, schema_path)) return SUCCESS_EXIT_CODE diff --git a/st2common/st2common/models/db/__init__.py b/st2common/st2common/models/db/__init__.py index a2c08c202d..7a34ad0370 100644 --- a/st2common/st2common/models/db/__init__.py +++ b/st2common/st2common/models/db/__init__.py @@ -126,7 +126,7 @@ def _db_connect(db_name, db_host, db_port, username=None, password=None, # NOTE: Since pymongo 3.0, connect() method is lazy and not blocking (always returns success) # so we need to issue a command / query to check if connection has been - # successfuly established. + # successfully established. # See http://api.mongodb.com/python/current/api/pymongo/mongo_client.html for details try: # The ismaster command is cheap and does not require auth diff --git a/st2reactor/st2reactor/container/process_container.py b/st2reactor/st2reactor/container/process_container.py index 33aaec95cf..1c9d0dc596 100644 --- a/st2reactor/st2reactor/container/process_container.py +++ b/st2reactor/st2reactor/container/process_container.py @@ -183,9 +183,10 @@ def _poll_sensors_for_results(self, sensor_ids): else: sensor_start_time = self._sensor_start_times[sensor_id] sensor_respawn_count = self._sensor_respawn_counts[sensor_id] - successfuly_started = (now - sensor_start_time) >= SENSOR_SUCCESSFUL_START_THRESHOLD + successfully_started = ((now - sensor_start_time) >= + SENSOR_SUCCESSFUL_START_THRESHOLD) - if successfuly_started and sensor_respawn_count >= 1: + if successfully_started and sensor_respawn_count >= 1: # Sensor has been successfully running more than threshold seconds, clear the # respawn counter so we can try to restart the sensor if it dies later on self._sensor_respawn_counts[sensor_id] = 0 @@ -402,7 +403,7 @@ def _respawn_sensor(self, sensor_id, sensor, exit_code): if self._single_sensor_mode: # In single sensor mode we want to exit immediately on failure LOG.info('Not respawning a sensor since running in single sensor mode', - extra=extra) + extra=extra) self._stopped = True self._exit_code = exit_code From c0dd93553a85595e60e951dea5c89924db72e3bf Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 30 Jul 2018 14:48:54 +0200 Subject: [PATCH 17/24] Move get_and_set_proxy config function in pack_management module so it can be re-used by other code. Also re-use existing do_register_cli_opts instead of copy and pasting it over. --- st2common/st2common/cmd/install_pack.py | 46 ++------------------- st2common/st2common/config.py | 7 ++++ st2common/st2common/util/pack_management.py | 37 ++++++++++++++++- 3 files changed, 46 insertions(+), 44 deletions(-) diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py index bcaf1135bc..c65dc27941 100644 --- a/st2common/st2common/cmd/install_pack.py +++ b/st2common/st2common/cmd/install_pack.py @@ -20,8 +20,10 @@ from st2common import config from st2common import log as logging +from st2common.config import do_register_cli_opts from st2common.script_setup import setup as common_setup from st2common.util.pack_management import download_pack +from st2common.util.pack_management import get_and_set_proxy_config from st2common.util.virtualenvs import setup_pack_virtualenv __all__ = [ @@ -31,15 +33,6 @@ LOG = logging.getLogger(__name__) -def _do_register_cli_opts(opts, ignore_errors=False): - for opt in opts: - try: - cfg.CONF.register_cli_opt(opt) - except: - if not ignore_errors: - raise - - def _register_cli_opts(): cli_opts = [ cfg.MultiStrOpt('pack', default=None, required=True, positional=True, @@ -51,40 +44,7 @@ def _register_cli_opts(): help='True to force pack installation and ignore install ' 'lock file if it exists.'), ] - _do_register_cli_opts(cli_opts) - - -def get_and_set_proxy_config(): - https_proxy = os.environ.get('https_proxy', None) - http_proxy = os.environ.get('http_proxy', None) - proxy_ca_bundle_path = os.environ.get('proxy_ca_bundle_path', None) - no_proxy = os.environ.get('no_proxy', None) - - proxy_config = {} - - if http_proxy or https_proxy: - LOG.debug('Using proxy %s', http_proxy if http_proxy else https_proxy) - - proxy_config = { - 'https_proxy': https_proxy, - 'http_proxy': http_proxy, - 'proxy_ca_bundle_path': proxy_ca_bundle_path, - 'no_proxy': no_proxy - } - - if https_proxy and not os.environ.get('https_proxy', None): - os.environ['https_proxy'] = https_proxy - - if http_proxy and not os.environ.get('http_proxy', None): - os.environ['http_proxy'] = http_proxy - - if no_proxy and not os.environ.get('no_proxy', None): - os.environ['no_proxy'] = no_proxy - - if proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None): - os.environ['no_proxy'] = no_proxy - - return proxy_config + do_register_cli_opts(cli_opts) def main(argv): diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index d9f51c538d..a705de8388 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -25,6 +25,13 @@ from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH from st2common.constants.runners import PYTHON_RUNNER_DEFAULT_LOG_LEVEL +__all__ = [ + 'do_register_opts', + 'do_register_cli_opts', + + 'parse_args' +] + def do_register_opts(opts, group=None, ignore_errors=False): try: diff --git a/st2common/st2common/util/pack_management.py b/st2common/st2common/util/pack_management.py index 35cc77d219..0856f56af5 100644 --- a/st2common/st2common/util/pack_management.py +++ b/st2common/st2common/util/pack_management.py @@ -47,7 +47,9 @@ 'download_pack', 'get_repo_url', - 'eval_repo_url' + 'eval_repo_url', + + 'get_and_set_proxy_config' ] LOG = logging.getLogger(__name__) @@ -379,3 +381,36 @@ def get_pack_ref(pack_dir): pack_ref = get_pack_ref_from_metadata(metadata=metadata, pack_directory_name=None) return pack_ref + + +def get_and_set_proxy_config(): + https_proxy = os.environ.get('https_proxy', None) + http_proxy = os.environ.get('http_proxy', None) + proxy_ca_bundle_path = os.environ.get('proxy_ca_bundle_path', None) + no_proxy = os.environ.get('no_proxy', None) + + proxy_config = {} + + if http_proxy or https_proxy: + LOG.debug('Using proxy %s', http_proxy if http_proxy else https_proxy) + + proxy_config = { + 'https_proxy': https_proxy, + 'http_proxy': http_proxy, + 'proxy_ca_bundle_path': proxy_ca_bundle_path, + 'no_proxy': no_proxy + } + + if https_proxy and not os.environ.get('https_proxy', None): + os.environ['https_proxy'] = https_proxy + + if http_proxy and not os.environ.get('http_proxy', None): + os.environ['http_proxy'] = http_proxy + + if no_proxy and not os.environ.get('no_proxy', None): + os.environ['no_proxy'] = no_proxy + + if proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None): + os.environ['no_proxy'] = no_proxy + + return proxy_config From 04fe9cfc2a99b2d5a02fbe6e9d5f311cfcdce798 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 30 Jul 2018 14:57:00 +0200 Subject: [PATCH 18/24] Update more of the existing code to re-use existing method instead of using copy and pasted version. --- st2common/st2common/cmd/purge_executions.py | 16 ++++++---------- .../st2common/cmd/purge_trigger_instances.py | 16 ++++++---------- st2common/st2common/cmd/validate_config.py | 4 ++-- st2reactor/st2reactor/cmd/rule_tester.py | 12 ++---------- 4 files changed, 16 insertions(+), 32 deletions(-) diff --git a/st2common/st2common/cmd/purge_executions.py b/st2common/st2common/cmd/purge_executions.py index f3e2330a3a..15cea47cc9 100755 --- a/st2common/st2common/cmd/purge_executions.py +++ b/st2common/st2common/cmd/purge_executions.py @@ -29,22 +29,18 @@ from st2common import config from st2common import log as logging +from st2common.config import do_register_cli_opts from st2common.script_setup import setup as common_setup from st2common.script_setup import teardown as common_teardown from st2common.constants.exit_codes import SUCCESS_EXIT_CODE from st2common.constants.exit_codes import FAILURE_EXIT_CODE from st2common.garbage_collection.executions import purge_executions -LOG = logging.getLogger(__name__) - +__all__ = [ + 'main' +] -def _do_register_cli_opts(opts, ignore_errors=False): - for opt in opts: - try: - cfg.CONF.register_cli_opt(opt) - except: - if not ignore_errors: - raise +LOG = logging.getLogger(__name__) def _register_cli_opts(): @@ -60,7 +56,7 @@ def _register_cli_opts(): 'By default, only executions in completed states such as "succeeeded" ' + ', "failed", "canceled" and "timed_out" are deleted.'), ] - _do_register_cli_opts(cli_opts) + do_register_cli_opts(cli_opts) def main(): diff --git a/st2common/st2common/cmd/purge_trigger_instances.py b/st2common/st2common/cmd/purge_trigger_instances.py index d028de0a78..4fd0b7fb7c 100755 --- a/st2common/st2common/cmd/purge_trigger_instances.py +++ b/st2common/st2common/cmd/purge_trigger_instances.py @@ -29,22 +29,18 @@ from st2common import config from st2common import log as logging +from st2common.config import do_register_cli_opts from st2common.script_setup import setup as common_setup from st2common.script_setup import teardown as common_teardown from st2common.constants.exit_codes import SUCCESS_EXIT_CODE from st2common.constants.exit_codes import FAILURE_EXIT_CODE from st2common.garbage_collection.trigger_instances import purge_trigger_instances -LOG = logging.getLogger(__name__) - +__all__ = [ + 'main' +] -def _do_register_cli_opts(opts, ignore_errors=False): - for opt in opts: - try: - cfg.CONF.register_cli_opt(opt) - except: - if not ignore_errors: - raise +LOG = logging.getLogger(__name__) def _register_cli_opts(): @@ -54,7 +50,7 @@ def _register_cli_opts(): 'this UTC timestamp. ' + 'Example value: 2015-03-13T19:01:27.255542Z') ] - _do_register_cli_opts(cli_opts) + do_register_cli_opts(cli_opts) def main(): diff --git a/st2common/st2common/cmd/validate_config.py b/st2common/st2common/cmd/validate_config.py index d23e6d5ef7..bfdafd1ddc 100644 --- a/st2common/st2common/cmd/validate_config.py +++ b/st2common/st2common/cmd/validate_config.py @@ -23,6 +23,7 @@ from oslo_config import cfg +from st2common.config import do_register_cli_opts from st2common.constants.system import VERSION_STRING from st2common.constants.exit_codes import SUCCESS_EXIT_CODE from st2common.constants.exit_codes import FAILURE_EXIT_CODE @@ -50,8 +51,7 @@ def _register_cli_opts(): help='Path to the config file to validate.'), ] - for opt in cli_opts: - cfg.CONF.register_cli_opt(opt) + do_register_cli_opts(cli_opts) def main(): diff --git a/st2reactor/st2reactor/cmd/rule_tester.py b/st2reactor/st2reactor/cmd/rule_tester.py index 7354f3ecab..6ba466ea31 100644 --- a/st2reactor/st2reactor/cmd/rule_tester.py +++ b/st2reactor/st2reactor/cmd/rule_tester.py @@ -20,6 +20,7 @@ from st2common import config from st2common import log as logging +from st2common.config import do_register_cli_opts from st2common.script_setup import setup as common_setup from st2common.script_setup import teardown as common_teardown from st2reactor.rules.tester import RuleTester @@ -31,15 +32,6 @@ LOG = logging.getLogger(__name__) -def _do_register_cli_opts(opts, ignore_errors=False): - for opt in opts: - try: - cfg.CONF.register_cli_opt(opt) - except: - if not ignore_errors: - raise - - def _register_cli_opts(): cli_opts = [ cfg.StrOpt('rule', default=None, @@ -51,7 +43,7 @@ def _register_cli_opts(): cfg.StrOpt('trigger-instance-id', default=None, help='Id of the Trigger Instance to use for validation.') ] - _do_register_cli_opts(cli_opts) + do_register_cli_opts(cli_opts) def main(): From ad7bd59514a24a996eb9fbff4a8f71329f01bb23 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 30 Jul 2018 14:58:40 +0200 Subject: [PATCH 19/24] Remove unused import. --- st2common/st2common/cmd/install_pack.py | 1 - 1 file changed, 1 deletion(-) diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py index c65dc27941..9fb05428d0 100644 --- a/st2common/st2common/cmd/install_pack.py +++ b/st2common/st2common/cmd/install_pack.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import sys from oslo_config import cfg From 6e9668348e1f35f00fa8948525eb89ba9d22505b Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 30 Jul 2018 15:04:35 +0200 Subject: [PATCH 20/24] Add new st2-pack-download and st2-pack-setup-virtualenv CLI commands. Those commands are analogous of packs.download and packs.setup_virtualenv StackStorm actions. Running st2-pack-install is the same as running st2-pack-download first and st2-pack-setup-virtualenv second. --- st2common/bin/st2-pack-download | 27 +++++++ st2common/bin/st2-pack-setup-virtualenv | 27 +++++++ st2common/st2common/cmd/download_pack.py | 77 +++++++++++++++++++ .../st2common/cmd/setup_pack_virtualenv.py | 69 +++++++++++++++++ 4 files changed, 200 insertions(+) create mode 100755 st2common/bin/st2-pack-download create mode 100755 st2common/bin/st2-pack-setup-virtualenv create mode 100644 st2common/st2common/cmd/download_pack.py create mode 100644 st2common/st2common/cmd/setup_pack_virtualenv.py diff --git a/st2common/bin/st2-pack-download b/st2common/bin/st2-pack-download new file mode 100755 index 0000000000..137c15c121 --- /dev/null +++ b/st2common/bin/st2-pack-download @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""" +StackStorm command which downloads the pack and places it in the local content +repository. +""" + +import sys + +from st2common.cmd import download_pack + +if __name__ == '__main__': + sys.exit(download_pack.main(sys.argv[1:])) diff --git a/st2common/bin/st2-pack-setup-virtualenv b/st2common/bin/st2-pack-setup-virtualenv new file mode 100755 index 0000000000..7c13286965 --- /dev/null +++ b/st2common/bin/st2-pack-setup-virtualenv @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""" +StackStorm command which downloads the pack and places it in the local content +repository. +""" + +import sys + +from st2common.cmd import setup_pack_virtualenv + +if __name__ == '__main__': + sys.exit(setup_pack_virtualenv.main(sys.argv[1:])) diff --git a/st2common/st2common/cmd/download_pack.py b/st2common/st2common/cmd/download_pack.py new file mode 100644 index 0000000000..7765b95c35 --- /dev/null +++ b/st2common/st2common/cmd/download_pack.py @@ -0,0 +1,77 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from oslo_config import cfg + +from st2common import config +from st2common import log as logging +from st2common.config import do_register_cli_opts +from st2common.script_setup import setup as common_setup +from st2common.util.pack_management import download_pack +from st2common.util.pack_management import get_and_set_proxy_config + +__all__ = [ + 'main' +] + +LOG = logging.getLogger(__name__) + + +def _register_cli_opts(): + cli_opts = [ + cfg.MultiStrOpt('pack', default=None, required=True, positional=True, + help='Name of the pack to install (download).'), + cfg.BoolOpt('verify-ssl', default=True, + help=('Verify SSL certificate of the Git repo from which the pack is ' + 'installed.')), + cfg.BoolOpt('force', default=False, + help='True to force pack download and ignore download ' + 'lock file if it exists.'), + ] + do_register_cli_opts(cli_opts) + + +def main(argv): + _register_cli_opts() + + # Parse CLI args, set up logging + common_setup(config=config, setup_db=False, register_mq_exchanges=False, + register_internal_trigger_types=False) + + packs = cfg.CONF.pack + verify_ssl = cfg.CONF.verify_ssl + force = cfg.CONF.force + + proxy_config = get_and_set_proxy_config() + + for pack in packs: + LOG.info('Installing pack "%s"' % (pack)) + result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, + proxy_config=proxy_config, force_permissions=True) + + # Raw pack name excluding the version + pack_name = result[1] + success = result[2][0] + + if success: + LOG.info('Successfully installed pack "%s"' % (pack_name)) + else: + error = result[2][1] + LOG.error('Failed to installed pack "%s": %s' % (pack_name, error)) + sys.exit(2) + + return 0 diff --git a/st2common/st2common/cmd/setup_pack_virtualenv.py b/st2common/st2common/cmd/setup_pack_virtualenv.py new file mode 100644 index 0000000000..f4218b9b2d --- /dev/null +++ b/st2common/st2common/cmd/setup_pack_virtualenv.py @@ -0,0 +1,69 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +from st2common import config +from st2common import log as logging +from st2common.config import do_register_cli_opts +from st2common.script_setup import setup as common_setup +from st2common.util.pack_management import get_and_set_proxy_config +from st2common.util.virtualenvs import setup_pack_virtualenv + +__all__ = [ + 'main' +] + +LOG = logging.getLogger(__name__) + + +def _register_cli_opts(): + cli_opts = [ + cfg.MultiStrOpt('pack', default=None, required=True, positional=True, + help='Name of the pack to setup the virtual environment for.'), + cfg.BoolOpt('update', default=False, + help=('Check this option if the virtual environment already exists and if you ' + 'only want to perform an update and installation of new dependencies. If ' + 'you don\'t check this option, the virtual environment will be destroyed ' + 'then re-created. If you check this and the virtual environment doesn\'t ' + 'exist, it will create it..')), + cfg.BoolOpt('python3', default=False, + help='Use Python 3 binary when creating a virtualenv for this pack.'), + ] + do_register_cli_opts(cli_opts) + + +def main(argv): + _register_cli_opts() + + # Parse CLI args, set up logging + common_setup(config=config, setup_db=False, register_mq_exchanges=False, + register_internal_trigger_types=False) + + packs = cfg.CONF.pack + update = cfg.CONF.update + use_python3 = cfg.CONF.python3 + + proxy_config = get_and_set_proxy_config() + + for pack in packs: + # Setup pack virtual environment + LOG.info('Setting up virtualenv for pack "%s"' % (pack)) + setup_pack_virtualenv(pack_name=pack, update=update, logger=LOG, + proxy_config=proxy_config, use_python3=use_python3, + no_download=True) + LOG.info('Successfully set up virtualenv for pack "%s"' % (pack)) + + return 0 From ac123a028f296d49e09dda35efb3bb458fa0d161 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 30 Jul 2018 15:18:18 +0200 Subject: [PATCH 21/24] Update affected code. --- st2common/st2common/cmd/install_pack.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py index 9fb05428d0..523e5b439d 100644 --- a/st2common/st2common/cmd/install_pack.py +++ b/st2common/st2common/cmd/install_pack.py @@ -65,22 +65,20 @@ def main(argv): result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, proxy_config=proxy_config, force_permissions=True) - # Raw pack name excluding the version - pack_name = result[1] success = result[2][0] if success: - LOG.info('Successfully installed pack "%s"' % (pack_name)) + LOG.info('Successfuly installed pack "%s"' % (pack)) else: error = result[2][1] - LOG.error('Failed to install pack "%s": %s' % (pack_name, error)) + LOG.error('Failed to install pack "%s": %s' % (pack, error)) sys.exit(2) # 2. Setup pack virtual environment - LOG.info('Setting up virtualenv for pack "%s"' % (pack_name)) - setup_pack_virtualenv(pack_name=pack_name, update=False, logger=LOG, + LOG.info('Setting up virtualenv for pack "%s"' % (pack)) + setup_pack_virtualenv(pack_name=pack, update=False, logger=LOG, proxy_config=proxy_config, use_python3=False, no_download=True) - LOG.info('Successfully set up virtualenv for pack "%s"' % (pack_name)) + LOG.info('Successfuly set up virtualenv for pack "%s"' % (pack)) return 0 From edd94ece9b883a678e43ae011edd6c45afbf9055 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 30 Jul 2018 17:13:26 +0200 Subject: [PATCH 22/24] Fix typo. --- st2common/st2common/cmd/install_pack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py index 523e5b439d..7001c2ee26 100644 --- a/st2common/st2common/cmd/install_pack.py +++ b/st2common/st2common/cmd/install_pack.py @@ -68,7 +68,7 @@ def main(argv): success = result[2][0] if success: - LOG.info('Successfuly installed pack "%s"' % (pack)) + LOG.info('Successfully installed pack "%s"' % (pack)) else: error = result[2][1] LOG.error('Failed to install pack "%s": %s' % (pack, error)) @@ -79,6 +79,6 @@ def main(argv): setup_pack_virtualenv(pack_name=pack, update=False, logger=LOG, proxy_config=proxy_config, use_python3=False, no_download=True) - LOG.info('Successfuly set up virtualenv for pack "%s"' % (pack)) + LOG.info('Successfully set up virtualenv for pack "%s"' % (pack)) return 0 From 48a243d48c9eb2a1165ff4cadf26313c6a3659c1 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 30 Jul 2018 17:16:07 +0200 Subject: [PATCH 23/24] Add back changes which were accidentaly removed in ac123a028f296d49e09dda35efb3bb458fa0d161. --- st2common/st2common/cmd/install_pack.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py index 7001c2ee26..9fb05428d0 100644 --- a/st2common/st2common/cmd/install_pack.py +++ b/st2common/st2common/cmd/install_pack.py @@ -65,20 +65,22 @@ def main(argv): result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, proxy_config=proxy_config, force_permissions=True) + # Raw pack name excluding the version + pack_name = result[1] success = result[2][0] if success: - LOG.info('Successfully installed pack "%s"' % (pack)) + LOG.info('Successfully installed pack "%s"' % (pack_name)) else: error = result[2][1] - LOG.error('Failed to install pack "%s": %s' % (pack, error)) + LOG.error('Failed to install pack "%s": %s' % (pack_name, error)) sys.exit(2) # 2. Setup pack virtual environment - LOG.info('Setting up virtualenv for pack "%s"' % (pack)) - setup_pack_virtualenv(pack_name=pack, update=False, logger=LOG, + LOG.info('Setting up virtualenv for pack "%s"' % (pack_name)) + setup_pack_virtualenv(pack_name=pack_name, update=False, logger=LOG, proxy_config=proxy_config, use_python3=False, no_download=True) - LOG.info('Successfully set up virtualenv for pack "%s"' % (pack)) + LOG.info('Successfully set up virtualenv for pack "%s"' % (pack_name)) return 0 From 5513b7d73f137e68290aa06c3068875535b23728 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 31 Jul 2018 11:27:11 +0200 Subject: [PATCH 24/24] Update "setup_virtualenv" function so it also sets / changes owner group of the pack virtualenv directory to the value defined in the config (content.pack_group, defaults to st2packs). This way the end result is consistent also when running st2-pack-setup-virtualenv and st2ctl reload --register-setup-virtualenvs command as a user which is not part of st2packs group. Keep in mind that this step requires sudo access, but if sudo is not available, or group ownership change step fails, the command itself doesn't fail. This means that sudo access for st2-pack-{download,setup-virtualenv,install} commands is optional. --- st2common/st2common/util/pack_management.py | 43 +++++++++++++++++---- st2common/st2common/util/virtualenvs.py | 7 +++- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/st2common/st2common/util/pack_management.py b/st2common/st2common/util/pack_management.py index 0856f56af5..d9a7346e84 100644 --- a/st2common/st2common/util/pack_management.py +++ b/st2common/st2common/util/pack_management.py @@ -49,6 +49,9 @@ 'get_repo_url', 'eval_repo_url', + 'apply_pack_owner_group', + 'apply_pack_permissions', + 'get_and_set_proxy_config' ] @@ -59,7 +62,7 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, force=False, - proxy_config=None, force_permissions=True, logger=LOG): + proxy_config=None, force_owner_group=True, force_permissions=True, logger=LOG): """ Download the pack and move it to /opt/stackstorm/packs. @@ -69,6 +72,10 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, :param pack: Pack name. :rtype pack: ``str`` + :param force_owner_group: Set owner group of the pack directory to the value defined in the + config. + :type force_owner_group: ``bool`` + :param force_permissions: True to force 770 permission on all the pack content. :type force_permissions: ``bool`` @@ -121,6 +128,7 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, # 3. Move pack to the final location move_result = move_pack(abs_repo_base=abs_repo_base, pack_name=pack_ref, abs_local_path=abs_local_path, + force_owner_group=force_owner_group, force_permissions=force_permissions, logger=logger) result[2] = move_result @@ -199,7 +207,8 @@ def clone_repo(temp_dir, repo_url, verify_ssl=True, ref='master'): return temp_dir -def move_pack(abs_repo_base, pack_name, abs_local_path, force_permissions=True, logger=LOG): +def move_pack(abs_repo_base, pack_name, abs_local_path, force_owner_group=True, + force_permissions=True, logger=LOG): """ Move pack directory into the final location. """ @@ -225,8 +234,14 @@ def move_pack(abs_repo_base, pack_name, abs_local_path, force_permissions=True, shutil.move(abs_local_path, dest_pack_path) # post move fix all permissions + if force_owner_group: + # 1. switch owner group to configured group + apply_pack_owner_group(pack_path=dest_pack_path) + if force_permissions: + # 2. Setup the right permissions and group ownership apply_pack_permissions(pack_path=dest_pack_path) + message = 'Success.' elif message: message = 'Failure : %s' % message @@ -234,17 +249,31 @@ def move_pack(abs_repo_base, pack_name, abs_local_path, force_permissions=True, return (desired, message) -def apply_pack_permissions(pack_path): +def apply_pack_owner_group(pack_path): """ - Recursively apply permission 770 to pack and its contents. + Switch owner group of the pack / virtualenv directory to the configured + group. + + NOTE: This requires sudo access. """ - # 1. switch owner group to configured group pack_group = utils.get_pack_group() if pack_group: - shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path]) + LOG.debug('Changing owner group of "%s" directory to %s' % (pack_path, pack_group)) + exit_code, _, stderr, _ = shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path]) + + if exit_code != 0: + # Non fatal, but we still log it + LOG.debug('Failed to change owner group on directory "%s" to "%s": %s' % + (pack_path, pack_group, stderr)) - # 2. Setup the right permissions and group ownership + return True + + +def apply_pack_permissions(pack_path): + """ + Recursively apply permission 770 to pack and its contents. + """ # These mask is same as mode = 775 mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH os.chmod(pack_path, mode) diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py index 4c5327fbfb..00d2b63f24 100644 --- a/st2common/st2common/util/virtualenvs.py +++ b/st2common/st2common/util/virtualenvs.py @@ -31,6 +31,7 @@ from st2common.util.shell import run_command from st2common.util.shell import quote_unix from st2common.util.compat import to_ascii +from st2common.util.pack_management import apply_pack_owner_group from st2common.content.utils import get_packs_base_paths from st2common.content.utils import get_pack_directory @@ -43,7 +44,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True, include_setuptools=True, include_wheel=True, proxy_config=None, - use_python3=False, no_download=True): + use_python3=False, no_download=True, force_owner_group=True): """ Setup virtual environment for the provided pack. @@ -121,6 +122,10 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True else: logger.debug('No pack specific requirements found') + # 5. Set the owner group + if force_owner_group: + apply_pack_owner_group(pack_path=virtualenv_path) + action = 'updated' if update else 'created' logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' % (pack_name, action, virtualenv_path))