diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 30557ee36d..735ebdadb9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,18 @@ Added * Add new ``?tags``, query param filter to the ``/v1/actions`` API endpoint. This query parameter allows users to filter out actions based on the tag name . By default, when no filter values are provided, all actions are returned. (new feature) #4219 +* Add a new standalone standalone ``st2-pack-install`` CLI command. This command installs a pack + (and sets up the pack virtual environment) on the server where it runs. It doesn't register the + content. It only depends on the Python, git and pip binary and ``st2common`` Python package to be + installed on the system where it runs. It doesn't depend on the database (MongoDB) and message + bus (RabbitMQ). + + It's primary meant to be used in scenarios where the content (packs) are baked into the base + container / VM image which is deployed to the cluster. + + Keep in mind that the content itself still needs to be registered with StackStorm at some later + point when access to RabbitMQ and MongoDB is available by running + ``st2ctl reload --register-all``. (new feature) #3912 #4256 Changed ~~~~~~~ @@ -66,7 +78,7 @@ Changed * Migrated runners to using the ``in-requirements.txt`` pattern for "components" in the build system, so the ``Makefile`` correctly generates and installs runner dependencies during testing and packaging. (improvement) (bugfix) #4169 - + Contributed by Nick Maludy (Encore Technologies). * Update ``st2`` CLI to use a more sensible default terminal size for table formatting purposes if we are unable to retrieve terminal size using various system-specific approaches. diff --git a/contrib/packs/actions/pack_mgmt/download.py b/contrib/packs/actions/pack_mgmt/download.py index 5e6438a294..8869fca635 100644 --- a/contrib/packs/actions/pack_mgmt/download.py +++ b/contrib/packs/actions/pack_mgmt/download.py @@ -15,33 +15,15 @@ # limitations under the License. import os -import shutil -import hashlib -import stat -import re import six -from git.repo import Repo -from gitdb.exc import BadName, BadObject -from lockfile import LockFile from st2common.runners.base_action import Action -from st2common.content import utils -from st2common.constants.pack import MANIFEST_FILE_NAME -from st2common.constants.pack import PACK_RESERVED_CHARACTERS -from st2common.constants.pack import PACK_VERSION_SEPARATOR -from st2common.constants.pack import PACK_VERSION_REGEX -from st2common.services.packs import get_pack_from_index -from st2common.util.pack import get_pack_metadata -from st2common.util.pack import get_pack_ref_from_metadata -from st2common.util.green import shell -from st2common.util.versioning import complex_semver_match -from st2common.util.versioning import get_stackstorm_version +from st2common.util.pack_management import download_pack -CONFIG_FILE = 'config.yaml' - - -CURRENT_STACKSTROM_VERSION = get_stackstorm_version() +__all__ = [ + 'DownloadGitRepoAction' +] class DownloadGitRepoAction(Action): @@ -85,203 +67,21 @@ def run(self, packs, abs_repo_base, verifyssl=True, force=False): result = {} for pack in packs: - pack_url, pack_version = self._get_repo_url(pack, proxy_config=self.proxy_config) - - temp_dir_name = hashlib.md5(pack_url).hexdigest() - lock_file = LockFile('/tmp/%s' % (temp_dir_name)) - lock_file_path = lock_file.lock_file - - if force: - self.logger.debug('Force mode is enabled, deleting lock file...') - - try: - os.unlink(lock_file_path) - except OSError: - # Lock file doesn't exist or similar - pass - - with lock_file: - try: - user_home = os.path.expanduser('~') - abs_local_path = os.path.join(user_home, temp_dir_name) - self._clone_repo(temp_dir=abs_local_path, repo_url=pack_url, - verifyssl=verifyssl, ref=pack_version) - - pack_ref = self._get_pack_ref(abs_local_path) - - # Verify that the pack version if compatible with current StackStorm version - if not force: - self._verify_pack_version(pack_dir=abs_local_path) - - result[pack_ref] = self._move_pack(abs_repo_base, pack_ref, abs_local_path) - finally: - self._cleanup_repo(abs_local_path) + pack_result = download_pack(pack=pack, abs_repo_base=abs_repo_base, + verify_ssl=verifyssl, force=force, + proxy_config=self.proxy_config, + force_permissions=True, + logger=self.logger) + pack_url, pack_ref, pack_result = pack_result + result[pack_ref] = pack_result return self._validate_result(result=result, repo_url=pack_url) - @staticmethod - def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'): - # Switch to non-interactive mode - os.environ['GIT_TERMINAL_PROMPT'] = '0' - os.environ['GIT_ASKPASS'] = '/bin/echo' - - # Disable SSL cert checking if explictly asked - if not verifyssl: - os.environ['GIT_SSL_NO_VERIFY'] = 'true' - - # Clone the repo from git; we don't use shallow copying - # because we want the user to work with the repo in the - # future. - repo = Repo.clone_from(repo_url, temp_dir) - active_branch = repo.active_branch - - use_branch = False - - # Special case when a default repo branch is not "master" - # No ref provided so we just use a default active branch - if (not ref or ref == active_branch.name) and repo.active_branch.object == repo.head.commit: - gitref = repo.active_branch.object - else: - # Try to match the reference to a branch name (i.e. "master") - gitref = DownloadGitRepoAction._get_gitref(repo, 'origin/%s' % ref) - if gitref: - use_branch = True - - # Try to match the reference to a commit hash, a tag, or "master" - if not gitref: - gitref = DownloadGitRepoAction._get_gitref(repo, ref) - - # Try to match the reference to a "vX.Y.Z" tag - if not gitref and re.match(PACK_VERSION_REGEX, ref): - gitref = DownloadGitRepoAction._get_gitref(repo, 'v%s' % ref) - - # Giving up ¯\_(ツ)_/¯ - if not gitref: - format_values = [ref, repo_url] - msg = '"%s" is not a valid version, hash, tag or branch in %s.' - - valid_versions = DownloadGitRepoAction._get_valid_versions_for_repo(repo=repo) - if len(valid_versions) >= 1: - valid_versions_string = ', '.join(valid_versions) - - msg += ' Available versions are: %s.' - format_values.append(valid_versions_string) - - raise ValueError(msg % tuple(format_values)) - - # We're trying to figure out which branch the ref is actually on, - # since there's no direct way to check for this in git-python. - branches = repo.git.branch('-a', '--contains', gitref.hexsha) # pylint: disable=no-member - branches = branches.replace('*', '').split() - - if active_branch.name not in branches or use_branch: - branch = 'origin/%s' % ref if use_branch else branches[0] - short_branch = ref if use_branch else branches[0].split('/')[-1] - repo.git.checkout('-b', short_branch, branch) - branch = repo.head.reference - else: - branch = repo.active_branch.name - - repo.git.checkout(gitref.hexsha) # pylint: disable=no-member - repo.git.branch('-f', branch, gitref.hexsha) # pylint: disable=no-member - repo.git.checkout(branch) - - return temp_dir - - def _move_pack(self, abs_repo_base, pack_name, abs_local_path): - desired, message = DownloadGitRepoAction._is_desired_pack(abs_local_path, pack_name) - if desired: - to = abs_repo_base - dest_pack_path = os.path.join(abs_repo_base, pack_name) - if os.path.exists(dest_pack_path): - self.logger.debug('Removing existing pack %s in %s to replace.', pack_name, - dest_pack_path) - - # Ensure to preserve any existing configuration - old_config_file = os.path.join(dest_pack_path, CONFIG_FILE) - new_config_file = os.path.join(abs_local_path, CONFIG_FILE) - - if os.path.isfile(old_config_file): - shutil.move(old_config_file, new_config_file) - - shutil.rmtree(dest_pack_path) - - self.logger.debug('Moving pack from %s to %s.', abs_local_path, to) - shutil.move(abs_local_path, dest_pack_path) - # post move fix all permissions. - self._apply_pack_permissions(pack_path=dest_pack_path) - message = 'Success.' - elif message: - message = 'Failure : %s' % message - - return (desired, message) - - def _apply_pack_permissions(self, pack_path): - """ - Will recursively apply permission 770 to pack and its contents. - """ - # 1. switch owner group to configured group - pack_group = utils.get_pack_group() - if pack_group: - shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path]) - - # 2. Setup the right permissions and group ownership - # These mask is same as mode = 775 - mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH - os.chmod(pack_path, mode) - - # Yuck! Since os.chmod does not support chmod -R walk manually. - for root, dirs, files in os.walk(pack_path): - for d in dirs: - os.chmod(os.path.join(root, d), mode) - for f in files: - os.chmod(os.path.join(root, f), mode) - - @staticmethod - def _verify_pack_version(pack_dir): - pack_metadata = DownloadGitRepoAction._get_pack_metadata(pack_dir=pack_dir) - pack_name = pack_metadata.get('name', None) - required_stackstorm_version = pack_metadata.get('stackstorm_version', None) - - # If stackstorm_version attribute is speficied, verify that the pack works with currently - # running version of StackStorm - if required_stackstorm_version: - if not complex_semver_match(CURRENT_STACKSTROM_VERSION, required_stackstorm_version): - msg = ('Pack "%s" requires StackStorm "%s", but current version is "%s". ' % - (pack_name, required_stackstorm_version, CURRENT_STACKSTROM_VERSION), - 'You can override this restriction by providing the "force" flag, but ', - 'the pack is not guaranteed to work.') - raise ValueError(msg) - - @staticmethod - def _is_desired_pack(abs_pack_path, pack_name): - # path has to exist. - if not os.path.exists(abs_pack_path): - return (False, 'Pack "%s" not found or it\'s missing a "pack.yaml" file.' % - (pack_name)) - - # should not include reserved characters - for character in PACK_RESERVED_CHARACTERS: - if character in pack_name: - return (False, 'Pack name "%s" contains reserved character "%s"' % - (pack_name, character)) - - # must contain a manifest file. Empty file is ok for now. - if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE_NAME)): - return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE_NAME)) - - return (True, '') - - @staticmethod - def _cleanup_repo(abs_local_path): - # basic lock checking etc? - if os.path.isdir(abs_local_path): - shutil.rmtree(abs_local_path) - @staticmethod def _validate_result(result, repo_url): atleast_one_success = False sanitized_result = {} + for k, v in six.iteritems(result): atleast_one_success |= v[0] sanitized_result[k] = v[1] @@ -299,71 +99,3 @@ def _validate_result(result, repo_url): raise Exception(message) return sanitized_result - - @staticmethod - def _get_repo_url(pack, proxy_config=None): - pack_and_version = pack.split(PACK_VERSION_SEPARATOR) - name_or_url = pack_and_version[0] - version = pack_and_version[1] if len(pack_and_version) > 1 else None - - if len(name_or_url.split('/')) == 1: - pack = get_pack_from_index(name_or_url, proxy_config=proxy_config) - if not pack: - raise Exception('No record of the "%s" pack in the index.' % name_or_url) - return (pack['repo_url'], version) - else: - return (DownloadGitRepoAction._eval_repo_url(name_or_url), version) - - @staticmethod - def _eval_repo_url(repo_url): - """Allow passing short GitHub style URLs""" - if not repo_url: - raise Exception('No valid repo_url provided or could be inferred.') - if repo_url.startswith("file://"): - return repo_url - else: - if len(repo_url.split('/')) == 2 and 'git@' not in repo_url: - url = 'https://github.com/{}'.format(repo_url) - else: - url = repo_url - return url - - @staticmethod - def _get_pack_metadata(pack_dir): - metadata = get_pack_metadata(pack_dir=pack_dir) - return metadata - - @staticmethod - def _get_pack_ref(pack_dir): - """ - Read pack name from the metadata file and sanitize it. - """ - metadata = DownloadGitRepoAction._get_pack_metadata(pack_dir=pack_dir) - pack_ref = get_pack_ref_from_metadata(metadata=metadata, - pack_directory_name=None) - return pack_ref - - @staticmethod - def _get_valid_versions_for_repo(repo): - """ - Method which returns a valid versions for a particular repo (pack). - - It does so by introspecting available tags. - - :rtype: ``list`` of ``str`` - """ - valid_versions = [] - - for tag in repo.tags: - if tag.name.startswith('v') and re.match(PACK_VERSION_REGEX, tag.name[1:]): - # Note: We strip leading "v" from the version number - valid_versions.append(tag.name[1:]) - - return valid_versions - - @staticmethod - def _get_gitref(repo, ref): - try: - return repo.commit(ref) - except (BadName, BadObject): - return False diff --git a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py index 1d45d044d1..b645ede90c 100644 --- a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py +++ b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py @@ -88,6 +88,6 @@ def run(self, packs, update=False, python3=False, no_download=True): proxy_config=self.proxy_config, use_python3=python3, no_download=no_download) - message = ('Successfuly set up virtualenv for the following packs: %s' % + message = ('Successfully set up virtualenv for the following packs: %s' % (', '.join(packs))) return message diff --git a/contrib/packs/tests/test_action_download.py b/contrib/packs/tests/test_action_download.py index c951089c09..f6b18332fb 100644 --- a/contrib/packs/tests/test_action_download.py +++ b/contrib/packs/tests/test_action_download.py @@ -31,7 +31,9 @@ from st2common.services import packs as pack_service from st2tests.base import BaseActionTestCase -import pack_mgmt.download +import st2common.util.pack_management +from st2common.util.pack_management import eval_repo_url + from pack_mgmt.download import DownloadGitRepoAction PACK_INDEX = { @@ -218,7 +220,7 @@ def side_effect(ref): self.assertEqual(result, {'test': 'Success.'}) - @mock.patch.object(DownloadGitRepoAction, '_get_valid_versions_for_repo', + @mock.patch.object(st2common.util.pack_management, 'get_valid_versions_for_repo', mock.Mock(return_value=['1.0.0', '2.0.0'])) def test_run_pack_download_invalid_version(self): self.repo_instance.commit.side_effect = lambda ref: None @@ -234,69 +236,69 @@ def test_download_pack_stackstorm_version_identifier_check(self): action = self.get_action_instance() # Version is satisfied - pack_mgmt.download.CURRENT_STACKSTROM_VERSION = '2.0.0' + st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '2.0.0' result = action.run(packs=['test3'], abs_repo_base=self.repo_base) self.assertEqual(result['test3'], 'Success.') # Pack requires a version which is not satisfied by current StackStorm version - pack_mgmt.download.CURRENT_STACKSTROM_VERSION = '2.2.0' + st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '2.2.0' expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' 'current version is "2.2.0"') self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], abs_repo_base=self.repo_base) - pack_mgmt.download.CURRENT_STACKSTROM_VERSION = '2.3.0' + st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '2.3.0' expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' 'current version is "2.3.0"') self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], abs_repo_base=self.repo_base) - pack_mgmt.download.CURRENT_STACKSTROM_VERSION = '1.5.9' + st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '1.5.9' expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' 'current version is "1.5.9"') self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], abs_repo_base=self.repo_base) - pack_mgmt.download.CURRENT_STACKSTROM_VERSION = '1.5.0' + st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '1.5.0' expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but ' 'current version is "1.5.0"') self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'], abs_repo_base=self.repo_base) # Version is not met, but force=true parameter is provided - pack_mgmt.download.CURRENT_STACKSTROM_VERSION = '1.5.0' + st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '1.5.0' result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=True) self.assertEqual(result['test3'], 'Success.') def test_resolve_urls(self): - url = DownloadGitRepoAction._eval_repo_url( + url = eval_repo_url( "https://github.com/StackStorm-Exchange/stackstorm-test") self.assertEqual(url, "https://github.com/StackStorm-Exchange/stackstorm-test") - url = DownloadGitRepoAction._eval_repo_url( + url = eval_repo_url( "https://github.com/StackStorm-Exchange/stackstorm-test.git") self.assertEqual(url, "https://github.com/StackStorm-Exchange/stackstorm-test.git") - url = DownloadGitRepoAction._eval_repo_url("StackStorm-Exchange/stackstorm-test") + url = eval_repo_url("StackStorm-Exchange/stackstorm-test") self.assertEqual(url, "https://github.com/StackStorm-Exchange/stackstorm-test") - url = DownloadGitRepoAction._eval_repo_url("git://StackStorm-Exchange/stackstorm-test") + url = eval_repo_url("git://StackStorm-Exchange/stackstorm-test") self.assertEqual(url, "git://StackStorm-Exchange/stackstorm-test") - url = DownloadGitRepoAction._eval_repo_url("git://StackStorm-Exchange/stackstorm-test.git") + url = eval_repo_url("git://StackStorm-Exchange/stackstorm-test.git") self.assertEqual(url, "git://StackStorm-Exchange/stackstorm-test.git") - url = DownloadGitRepoAction._eval_repo_url("git@github.com:foo/bar.git") + url = eval_repo_url("git@github.com:foo/bar.git") self.assertEqual(url, "git@github.com:foo/bar.git") - url = DownloadGitRepoAction._eval_repo_url("file:///home/vagrant/stackstorm-test") + url = eval_repo_url("file:///home/vagrant/stackstorm-test") self.assertEqual(url, "file:///home/vagrant/stackstorm-test") - url = DownloadGitRepoAction._eval_repo_url('ssh:///AutomationStackStorm') + url = eval_repo_url('ssh:///AutomationStackStorm') self.assertEqual(url, 'ssh:///AutomationStackStorm') - url = DownloadGitRepoAction._eval_repo_url('ssh://joe@local/AutomationStackStorm') + url = eval_repo_url('ssh://joe@local/AutomationStackStorm') self.assertEqual(url, 'ssh://joe@local/AutomationStackStorm') def test_run_pack_download_edge_cases(self): diff --git a/scripts/travis/install-and-run-mongodb.sh b/scripts/travis/install-and-run-mongodb.sh index fbe8abdc6f..3c2ae74934 100755 --- a/scripts/travis/install-and-run-mongodb.sh +++ b/scripts/travis/install-and-run-mongodb.sh @@ -46,7 +46,7 @@ MONGODB_PID=$! sleep 5 if ps -p ${MONGODB_PID} > /dev/null; then - echo "MongoDB successfuly started" + echo "MongoDB successfully started" tail -30 /tmp/mongodb.log exit 0 else diff --git a/st2common/bin/st2-pack-download b/st2common/bin/st2-pack-download new file mode 100755 index 0000000000..137c15c121 --- /dev/null +++ b/st2common/bin/st2-pack-download @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""" +StackStorm command which downloads the pack and places it in the local content +repository. +""" + +import sys + +from st2common.cmd import download_pack + +if __name__ == '__main__': + sys.exit(download_pack.main(sys.argv[1:])) diff --git a/st2common/bin/st2-pack-install b/st2common/bin/st2-pack-install new file mode 100755 index 0000000000..cb43364113 --- /dev/null +++ b/st2common/bin/st2-pack-install @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +StackStorm command for installing a pack and setting up pack virtual environment on the local +system where the command runs. + +This command is meant to be used in Docker and similar environments where container / server +where the pack is installed doesn't have access to the database (MongoDB) and message bus +(RabbitMQ). + +Keep in mind that pack still eventually needs to be registered in the database by running +"st2ctl reload --register-all" command on a server with access to the database and message bus. + +NOTE: Ideally for this command to work, whole StackStorm package would be installed on the +system, but at the very least, the following things need to be present: + + * Python 2.7 / 3.x with a recent virtualenv binary + * st2common StackStorm Python package + * /etc/st2/st2.conf config file (user can specify custom pack exchange index, settings there, + etc.) +""" + +import sys + +from st2common.cmd import install_pack + +if __name__ == '__main__': + sys.exit(install_pack.main(sys.argv[1:])) diff --git a/st2common/bin/st2-pack-setup-virtualenv b/st2common/bin/st2-pack-setup-virtualenv new file mode 100755 index 0000000000..7c13286965 --- /dev/null +++ b/st2common/bin/st2-pack-setup-virtualenv @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""" +StackStorm command which downloads the pack and places it in the local content +repository. +""" + +import sys + +from st2common.cmd import setup_pack_virtualenv + +if __name__ == '__main__': + sys.exit(setup_pack_virtualenv.main(sys.argv[1:])) diff --git a/st2common/setup.py b/st2common/setup.py index a00dc2c0cf..283fa5ddb9 100644 --- a/st2common/setup.py +++ b/st2common/setup.py @@ -61,7 +61,8 @@ 'bin/st2-self-check', 'bin/st2-track-result', 'bin/st2-validate-pack-config', - 'bin/st2-check-license' + 'bin/st2-check-license', + 'bin/st2-pack-install' ], entry_points={ 'st2common.metrics.driver': [ diff --git a/st2common/st2common/cmd/download_pack.py b/st2common/st2common/cmd/download_pack.py new file mode 100644 index 0000000000..7765b95c35 --- /dev/null +++ b/st2common/st2common/cmd/download_pack.py @@ -0,0 +1,77 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from oslo_config import cfg + +from st2common import config +from st2common import log as logging +from st2common.config import do_register_cli_opts +from st2common.script_setup import setup as common_setup +from st2common.util.pack_management import download_pack +from st2common.util.pack_management import get_and_set_proxy_config + +__all__ = [ + 'main' +] + +LOG = logging.getLogger(__name__) + + +def _register_cli_opts(): + cli_opts = [ + cfg.MultiStrOpt('pack', default=None, required=True, positional=True, + help='Name of the pack to install (download).'), + cfg.BoolOpt('verify-ssl', default=True, + help=('Verify SSL certificate of the Git repo from which the pack is ' + 'installed.')), + cfg.BoolOpt('force', default=False, + help='True to force pack download and ignore download ' + 'lock file if it exists.'), + ] + do_register_cli_opts(cli_opts) + + +def main(argv): + _register_cli_opts() + + # Parse CLI args, set up logging + common_setup(config=config, setup_db=False, register_mq_exchanges=False, + register_internal_trigger_types=False) + + packs = cfg.CONF.pack + verify_ssl = cfg.CONF.verify_ssl + force = cfg.CONF.force + + proxy_config = get_and_set_proxy_config() + + for pack in packs: + LOG.info('Installing pack "%s"' % (pack)) + result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, + proxy_config=proxy_config, force_permissions=True) + + # Raw pack name excluding the version + pack_name = result[1] + success = result[2][0] + + if success: + LOG.info('Successfully installed pack "%s"' % (pack_name)) + else: + error = result[2][1] + LOG.error('Failed to installed pack "%s": %s' % (pack_name, error)) + sys.exit(2) + + return 0 diff --git a/st2common/st2common/cmd/install_pack.py b/st2common/st2common/cmd/install_pack.py new file mode 100644 index 0000000000..9fb05428d0 --- /dev/null +++ b/st2common/st2common/cmd/install_pack.py @@ -0,0 +1,86 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from oslo_config import cfg + +from st2common import config +from st2common import log as logging +from st2common.config import do_register_cli_opts +from st2common.script_setup import setup as common_setup +from st2common.util.pack_management import download_pack +from st2common.util.pack_management import get_and_set_proxy_config +from st2common.util.virtualenvs import setup_pack_virtualenv + +__all__ = [ + 'main' +] + +LOG = logging.getLogger(__name__) + + +def _register_cli_opts(): + cli_opts = [ + cfg.MultiStrOpt('pack', default=None, required=True, positional=True, + help='Name of the pack to install.'), + cfg.BoolOpt('verify-ssl', default=True, + help=('Verify SSL certificate of the Git repo from which the pack is ' + 'downloaded.')), + cfg.BoolOpt('force', default=False, + help='True to force pack installation and ignore install ' + 'lock file if it exists.'), + ] + do_register_cli_opts(cli_opts) + + +def main(argv): + _register_cli_opts() + + # Parse CLI args, set up logging + common_setup(config=config, setup_db=False, register_mq_exchanges=False, + register_internal_trigger_types=False) + + packs = cfg.CONF.pack + verify_ssl = cfg.CONF.verify_ssl + force = cfg.CONF.force + + proxy_config = get_and_set_proxy_config() + + for pack in packs: + # 1. Download the pack + LOG.info('Installing pack "%s"' % (pack)) + result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, + proxy_config=proxy_config, force_permissions=True) + + # Raw pack name excluding the version + pack_name = result[1] + success = result[2][0] + + if success: + LOG.info('Successfully installed pack "%s"' % (pack_name)) + else: + error = result[2][1] + LOG.error('Failed to install pack "%s": %s' % (pack_name, error)) + sys.exit(2) + + # 2. Setup pack virtual environment + LOG.info('Setting up virtualenv for pack "%s"' % (pack_name)) + setup_pack_virtualenv(pack_name=pack_name, update=False, logger=LOG, + proxy_config=proxy_config, use_python3=False, + no_download=True) + LOG.info('Successfully set up virtualenv for pack "%s"' % (pack_name)) + + return 0 diff --git a/st2common/st2common/cmd/purge_executions.py b/st2common/st2common/cmd/purge_executions.py index f3e2330a3a..15cea47cc9 100755 --- a/st2common/st2common/cmd/purge_executions.py +++ b/st2common/st2common/cmd/purge_executions.py @@ -29,22 +29,18 @@ from st2common import config from st2common import log as logging +from st2common.config import do_register_cli_opts from st2common.script_setup import setup as common_setup from st2common.script_setup import teardown as common_teardown from st2common.constants.exit_codes import SUCCESS_EXIT_CODE from st2common.constants.exit_codes import FAILURE_EXIT_CODE from st2common.garbage_collection.executions import purge_executions -LOG = logging.getLogger(__name__) - +__all__ = [ + 'main' +] -def _do_register_cli_opts(opts, ignore_errors=False): - for opt in opts: - try: - cfg.CONF.register_cli_opt(opt) - except: - if not ignore_errors: - raise +LOG = logging.getLogger(__name__) def _register_cli_opts(): @@ -60,7 +56,7 @@ def _register_cli_opts(): 'By default, only executions in completed states such as "succeeeded" ' + ', "failed", "canceled" and "timed_out" are deleted.'), ] - _do_register_cli_opts(cli_opts) + do_register_cli_opts(cli_opts) def main(): diff --git a/st2common/st2common/cmd/purge_trigger_instances.py b/st2common/st2common/cmd/purge_trigger_instances.py index d028de0a78..4fd0b7fb7c 100755 --- a/st2common/st2common/cmd/purge_trigger_instances.py +++ b/st2common/st2common/cmd/purge_trigger_instances.py @@ -29,22 +29,18 @@ from st2common import config from st2common import log as logging +from st2common.config import do_register_cli_opts from st2common.script_setup import setup as common_setup from st2common.script_setup import teardown as common_teardown from st2common.constants.exit_codes import SUCCESS_EXIT_CODE from st2common.constants.exit_codes import FAILURE_EXIT_CODE from st2common.garbage_collection.trigger_instances import purge_trigger_instances -LOG = logging.getLogger(__name__) - +__all__ = [ + 'main' +] -def _do_register_cli_opts(opts, ignore_errors=False): - for opt in opts: - try: - cfg.CONF.register_cli_opt(opt) - except: - if not ignore_errors: - raise +LOG = logging.getLogger(__name__) def _register_cli_opts(): @@ -54,7 +50,7 @@ def _register_cli_opts(): 'this UTC timestamp. ' + 'Example value: 2015-03-13T19:01:27.255542Z') ] - _do_register_cli_opts(cli_opts) + do_register_cli_opts(cli_opts) def main(): diff --git a/st2common/st2common/cmd/setup_pack_virtualenv.py b/st2common/st2common/cmd/setup_pack_virtualenv.py new file mode 100644 index 0000000000..f4218b9b2d --- /dev/null +++ b/st2common/st2common/cmd/setup_pack_virtualenv.py @@ -0,0 +1,69 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +from st2common import config +from st2common import log as logging +from st2common.config import do_register_cli_opts +from st2common.script_setup import setup as common_setup +from st2common.util.pack_management import get_and_set_proxy_config +from st2common.util.virtualenvs import setup_pack_virtualenv + +__all__ = [ + 'main' +] + +LOG = logging.getLogger(__name__) + + +def _register_cli_opts(): + cli_opts = [ + cfg.MultiStrOpt('pack', default=None, required=True, positional=True, + help='Name of the pack to setup the virtual environment for.'), + cfg.BoolOpt('update', default=False, + help=('Check this option if the virtual environment already exists and if you ' + 'only want to perform an update and installation of new dependencies. If ' + 'you don\'t check this option, the virtual environment will be destroyed ' + 'then re-created. If you check this and the virtual environment doesn\'t ' + 'exist, it will create it..')), + cfg.BoolOpt('python3', default=False, + help='Use Python 3 binary when creating a virtualenv for this pack.'), + ] + do_register_cli_opts(cli_opts) + + +def main(argv): + _register_cli_opts() + + # Parse CLI args, set up logging + common_setup(config=config, setup_db=False, register_mq_exchanges=False, + register_internal_trigger_types=False) + + packs = cfg.CONF.pack + update = cfg.CONF.update + use_python3 = cfg.CONF.python3 + + proxy_config = get_and_set_proxy_config() + + for pack in packs: + # Setup pack virtual environment + LOG.info('Setting up virtualenv for pack "%s"' % (pack)) + setup_pack_virtualenv(pack_name=pack, update=update, logger=LOG, + proxy_config=proxy_config, use_python3=use_python3, + no_download=True) + LOG.info('Successfully set up virtualenv for pack "%s"' % (pack)) + + return 0 diff --git a/st2common/st2common/cmd/validate_config.py b/st2common/st2common/cmd/validate_config.py index 93aea16107..bfdafd1ddc 100644 --- a/st2common/st2common/cmd/validate_config.py +++ b/st2common/st2common/cmd/validate_config.py @@ -23,6 +23,7 @@ from oslo_config import cfg +from st2common.config import do_register_cli_opts from st2common.constants.system import VERSION_STRING from st2common.constants.exit_codes import SUCCESS_EXIT_CODE from st2common.constants.exit_codes import FAILURE_EXIT_CODE @@ -50,8 +51,7 @@ def _register_cli_opts(): help='Path to the config file to validate.'), ] - for opt in cli_opts: - cfg.CONF.register_cli_opt(opt) + do_register_cli_opts(cli_opts) def main(): @@ -76,5 +76,5 @@ def main(): print('Failed to validate pack config.\n%s' % str(e)) return FAILURE_EXIT_CODE - print('Config "%s" successfuly validated against schema in %s.' % (config_path, schema_path)) + print('Config "%s" successfully validated against schema in %s.' % (config_path, schema_path)) return SUCCESS_EXIT_CODE diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index d9f51c538d..a705de8388 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -25,6 +25,13 @@ from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH from st2common.constants.runners import PYTHON_RUNNER_DEFAULT_LOG_LEVEL +__all__ = [ + 'do_register_opts', + 'do_register_cli_opts', + + 'parse_args' +] + def do_register_opts(opts, group=None, ignore_errors=False): try: diff --git a/st2common/st2common/models/db/__init__.py b/st2common/st2common/models/db/__init__.py index a2c08c202d..7a34ad0370 100644 --- a/st2common/st2common/models/db/__init__.py +++ b/st2common/st2common/models/db/__init__.py @@ -126,7 +126,7 @@ def _db_connect(db_name, db_host, db_port, username=None, password=None, # NOTE: Since pymongo 3.0, connect() method is lazy and not blocking (always returns success) # so we need to issue a command / query to check if connection has been - # successfuly established. + # successfully established. # See http://api.mongodb.com/python/current/api/pymongo/mongo_client.html for details try: # The ismaster command is cheap and does not require auth diff --git a/st2common/st2common/util/pack_management.py b/st2common/st2common/util/pack_management.py new file mode 100644 index 0000000000..d9a7346e84 --- /dev/null +++ b/st2common/st2common/util/pack_management.py @@ -0,0 +1,445 @@ +# -*- coding: utf-8 -*- +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module containing pack management related functions. +""" + +from __future__ import absolute_import + +import os +import shutil +import hashlib +import stat +import re + +from git.repo import Repo +from gitdb.exc import BadName, BadObject +from lockfile import LockFile + +from st2common import log as logging +from st2common.content import utils +from st2common.constants.pack import MANIFEST_FILE_NAME +from st2common.constants.pack import PACK_RESERVED_CHARACTERS +from st2common.constants.pack import PACK_VERSION_SEPARATOR +from st2common.constants.pack import PACK_VERSION_REGEX +from st2common.services.packs import get_pack_from_index +from st2common.util.pack import get_pack_metadata +from st2common.util.pack import get_pack_ref_from_metadata +from st2common.util.green import shell +from st2common.util.versioning import complex_semver_match +from st2common.util.versioning import get_stackstorm_version + +__all__ = [ + 'download_pack', + + 'get_repo_url', + 'eval_repo_url', + + 'apply_pack_owner_group', + 'apply_pack_permissions', + + 'get_and_set_proxy_config' +] + +LOG = logging.getLogger(__name__) + +CONFIG_FILE = 'config.yaml' +CURRENT_STACKSTROM_VERSION = get_stackstorm_version() + + +def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, force=False, + proxy_config=None, force_owner_group=True, force_permissions=True, logger=LOG): + """ + Download the pack and move it to /opt/stackstorm/packs. + + :param abs_repo_base: Path where the pack should be installed to. + :type abs_repo_base: ``str`` + + :param pack: Pack name. + :rtype pack: ``str`` + + :param force_owner_group: Set owner group of the pack directory to the value defined in the + config. + :type force_owner_group: ``bool`` + + :param force_permissions: True to force 770 permission on all the pack content. + :type force_permissions: ``bool`` + + :param force: Force the installation and ignore / delete the lock file if it already exists. + :type force: ``bool`` + + :return: (pack_url, pack_ref, result) + :rtype: tuple + """ + proxy_config = proxy_config or {} + + try: + pack_url, pack_version = get_repo_url(pack, proxy_config=proxy_config) + except Exception as e: + # Pack not found or similar + result = [None, pack, (False, str(e))] + return result + + result = [pack_url, None, None] + + temp_dir_name = hashlib.md5(pack_url).hexdigest() + lock_file = LockFile('/tmp/%s' % (temp_dir_name)) + lock_file_path = lock_file.lock_file + + if force: + logger.debug('Force mode is enabled, deleting lock file...') + + try: + os.unlink(lock_file_path) + except OSError: + # Lock file doesn't exist or similar + pass + + with lock_file: + try: + user_home = os.path.expanduser('~') + abs_local_path = os.path.join(user_home, temp_dir_name) + + # 1. Clone / download the repo + clone_repo(temp_dir=abs_local_path, repo_url=pack_url, verify_ssl=verify_ssl, + ref=pack_version) + + pack_ref = get_pack_ref(pack_dir=abs_local_path) + result[1] = pack_ref + + # 2. Verify that the pack version if compatible with current StackStorm version + if not force: + verify_pack_version(pack_dir=abs_local_path) + + # 3. Move pack to the final location + move_result = move_pack(abs_repo_base=abs_repo_base, pack_name=pack_ref, + abs_local_path=abs_local_path, + force_owner_group=force_owner_group, + force_permissions=force_permissions, + logger=logger) + result[2] = move_result + finally: + cleanup_repo(abs_local_path=abs_local_path) + + return tuple(result) + + +def clone_repo(temp_dir, repo_url, verify_ssl=True, ref='master'): + # Switch to non-interactive mode + os.environ['GIT_TERMINAL_PROMPT'] = '0' + os.environ['GIT_ASKPASS'] = '/bin/echo' + + # Disable SSL cert checking if explictly asked + if not verify_ssl: + os.environ['GIT_SSL_NO_VERIFY'] = 'true' + + # Clone the repo from git; we don't use shallow copying + # because we want the user to work with the repo in the + # future. + repo = Repo.clone_from(repo_url, temp_dir) + active_branch = repo.active_branch + + use_branch = False + + # Special case when a default repo branch is not "master" + # No ref provided so we just use a default active branch + if (not ref or ref == active_branch.name) and repo.active_branch.object == repo.head.commit: + gitref = repo.active_branch.object + else: + # Try to match the reference to a branch name (i.e. "master") + gitref = get_gitref(repo, 'origin/%s' % ref) + if gitref: + use_branch = True + + # Try to match the reference to a commit hash, a tag, or "master" + if not gitref: + gitref = get_gitref(repo, ref) + + # Try to match the reference to a "vX.Y.Z" tag + if not gitref and re.match(PACK_VERSION_REGEX, ref): + gitref = get_gitref(repo, 'v%s' % ref) + + # Giving up ¯\_(ツ)_/¯ + if not gitref: + format_values = [ref, repo_url] + msg = '"%s" is not a valid version, hash, tag or branch in %s.' + + valid_versions = get_valid_versions_for_repo(repo=repo) + if len(valid_versions) >= 1: + valid_versions_string = ', '.join(valid_versions) + + msg += ' Available versions are: %s.' + format_values.append(valid_versions_string) + + raise ValueError(msg % tuple(format_values)) + + # We're trying to figure out which branch the ref is actually on, + # since there's no direct way to check for this in git-python. + branches = repo.git.branch('-a', '--contains', gitref.hexsha) # pylint: disable=no-member + branches = branches.replace('*', '').split() + + if active_branch.name not in branches or use_branch: + branch = 'origin/%s' % ref if use_branch else branches[0] + short_branch = ref if use_branch else branches[0].split('/')[-1] + repo.git.checkout('-b', short_branch, branch) + branch = repo.head.reference + else: + branch = repo.active_branch.name + + repo.git.checkout(gitref.hexsha) # pylint: disable=no-member + repo.git.branch('-f', branch, gitref.hexsha) # pylint: disable=no-member + repo.git.checkout(branch) + + return temp_dir + + +def move_pack(abs_repo_base, pack_name, abs_local_path, force_owner_group=True, + force_permissions=True, logger=LOG): + """ + Move pack directory into the final location. + """ + desired, message = is_desired_pack(abs_local_path, pack_name) + + if desired: + to = abs_repo_base + dest_pack_path = os.path.join(abs_repo_base, pack_name) + if os.path.exists(dest_pack_path): + logger.debug('Removing existing pack %s in %s to replace.', pack_name, + dest_pack_path) + + # Ensure to preserve any existing configuration + old_config_file = os.path.join(dest_pack_path, CONFIG_FILE) + new_config_file = os.path.join(abs_local_path, CONFIG_FILE) + + if os.path.isfile(old_config_file): + shutil.move(old_config_file, new_config_file) + + shutil.rmtree(dest_pack_path) + + logger.debug('Moving pack from %s to %s.', abs_local_path, to) + shutil.move(abs_local_path, dest_pack_path) + + # post move fix all permissions + if force_owner_group: + # 1. switch owner group to configured group + apply_pack_owner_group(pack_path=dest_pack_path) + + if force_permissions: + # 2. Setup the right permissions and group ownership + apply_pack_permissions(pack_path=dest_pack_path) + + message = 'Success.' + elif message: + message = 'Failure : %s' % message + + return (desired, message) + + +def apply_pack_owner_group(pack_path): + """ + Switch owner group of the pack / virtualenv directory to the configured + group. + + NOTE: This requires sudo access. + """ + pack_group = utils.get_pack_group() + + if pack_group: + LOG.debug('Changing owner group of "%s" directory to %s' % (pack_path, pack_group)) + exit_code, _, stderr, _ = shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path]) + + if exit_code != 0: + # Non fatal, but we still log it + LOG.debug('Failed to change owner group on directory "%s" to "%s": %s' % + (pack_path, pack_group, stderr)) + + return True + + +def apply_pack_permissions(pack_path): + """ + Recursively apply permission 770 to pack and its contents. + """ + # These mask is same as mode = 775 + mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH + os.chmod(pack_path, mode) + + # Yuck! Since os.chmod does not support chmod -R walk manually. + for root, dirs, files in os.walk(pack_path): + for d in dirs: + os.chmod(os.path.join(root, d), mode) + for f in files: + os.chmod(os.path.join(root, f), mode) + + +def cleanup_repo(abs_local_path): + # basic lock checking etc? + if os.path.isdir(abs_local_path): + shutil.rmtree(abs_local_path) + + +# Utility functions +def get_repo_url(pack, proxy_config=None): + """ + Retrieve pack repo url. + + :rtype: ``str`` + + :return: (repo_url, version) + :rtype: tuple + """ + pack_and_version = pack.split(PACK_VERSION_SEPARATOR) + name_or_url = pack_and_version[0] + version = pack_and_version[1] if len(pack_and_version) > 1 else None + + if len(name_or_url.split('/')) == 1: + pack = get_pack_from_index(name_or_url, proxy_config=proxy_config) + + if not pack: + raise Exception('No record of the "%s" pack in the index.' % (name_or_url)) + + return (pack['repo_url'], version) + else: + return (eval_repo_url(name_or_url), version) + + +def eval_repo_url(repo_url): + """ + Allow passing short GitHub style URLs. + """ + if not repo_url: + raise Exception('No valid repo_url provided or could be inferred.') + + if repo_url.startswith("file://"): + return repo_url + else: + if len(repo_url.split('/')) == 2 and 'git@' not in repo_url: + url = 'https://github.com/{}'.format(repo_url) + else: + url = repo_url + return url + + +def is_desired_pack(abs_pack_path, pack_name): + # path has to exist. + if not os.path.exists(abs_pack_path): + return (False, 'Pack "%s" not found or it\'s missing a "pack.yaml" file.' % + (pack_name)) + + # should not include reserved characters + for character in PACK_RESERVED_CHARACTERS: + if character in pack_name: + return (False, 'Pack name "%s" contains reserved character "%s"' % + (pack_name, character)) + + # must contain a manifest file. Empty file is ok for now. + if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE_NAME)): + return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE_NAME)) + + return (True, '') + + +def verify_pack_version(pack_dir): + """ + Verify that the pack works with the currently running StackStorm version. + """ + pack_metadata = get_pack_metadata(pack_dir=pack_dir) + pack_name = pack_metadata.get('name', None) + required_stackstorm_version = pack_metadata.get('stackstorm_version', None) + + # If stackstorm_version attribute is speficied, verify that the pack works with currently + # running version of StackStorm + if required_stackstorm_version: + if not complex_semver_match(CURRENT_STACKSTROM_VERSION, required_stackstorm_version): + msg = ('Pack "%s" requires StackStorm "%s", but current version is "%s". ' % + (pack_name, required_stackstorm_version, CURRENT_STACKSTROM_VERSION), + 'You can override this restriction by providing the "force" flag, but ', + 'the pack is not guaranteed to work.') + raise ValueError(msg) + + return True + + +def get_gitref(repo, ref): + """ + Retrieve git repo reference if available. + """ + try: + return repo.commit(ref) + except (BadName, BadObject): + return False + + +def get_valid_versions_for_repo(repo): + """ + Retrieve valid versions (tags) for a particular repo (pack). + + It does so by introspecting available tags. + + :rtype: ``list`` of ``str`` + """ + valid_versions = [] + + for tag in repo.tags: + if tag.name.startswith('v') and re.match(PACK_VERSION_REGEX, tag.name[1:]): + # Note: We strip leading "v" from the version number + valid_versions.append(tag.name[1:]) + + return valid_versions + + +def get_pack_ref(pack_dir): + """ + Read pack reference from the metadata file and sanitize it. + """ + metadata = get_pack_metadata(pack_dir=pack_dir) + pack_ref = get_pack_ref_from_metadata(metadata=metadata, + pack_directory_name=None) + return pack_ref + + +def get_and_set_proxy_config(): + https_proxy = os.environ.get('https_proxy', None) + http_proxy = os.environ.get('http_proxy', None) + proxy_ca_bundle_path = os.environ.get('proxy_ca_bundle_path', None) + no_proxy = os.environ.get('no_proxy', None) + + proxy_config = {} + + if http_proxy or https_proxy: + LOG.debug('Using proxy %s', http_proxy if http_proxy else https_proxy) + + proxy_config = { + 'https_proxy': https_proxy, + 'http_proxy': http_proxy, + 'proxy_ca_bundle_path': proxy_ca_bundle_path, + 'no_proxy': no_proxy + } + + if https_proxy and not os.environ.get('https_proxy', None): + os.environ['https_proxy'] = https_proxy + + if http_proxy and not os.environ.get('http_proxy', None): + os.environ['http_proxy'] = http_proxy + + if no_proxy and not os.environ.get('no_proxy', None): + os.environ['no_proxy'] = no_proxy + + if proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None): + os.environ['no_proxy'] = no_proxy + + return proxy_config diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py index 4c5327fbfb..00d2b63f24 100644 --- a/st2common/st2common/util/virtualenvs.py +++ b/st2common/st2common/util/virtualenvs.py @@ -31,6 +31,7 @@ from st2common.util.shell import run_command from st2common.util.shell import quote_unix from st2common.util.compat import to_ascii +from st2common.util.pack_management import apply_pack_owner_group from st2common.content.utils import get_packs_base_paths from st2common.content.utils import get_pack_directory @@ -43,7 +44,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True, include_setuptools=True, include_wheel=True, proxy_config=None, - use_python3=False, no_download=True): + use_python3=False, no_download=True, force_owner_group=True): """ Setup virtual environment for the provided pack. @@ -121,6 +122,10 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True else: logger.debug('No pack specific requirements found') + # 5. Set the owner group + if force_owner_group: + apply_pack_owner_group(pack_path=virtualenv_path) + action = 'updated' if update else 'created' logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' % (pack_name, action, virtualenv_path)) diff --git a/st2common/tests/unit/test_pack_management.py b/st2common/tests/unit/test_pack_management.py index 0f15b49a60..b7be63f670 100644 --- a/st2common/tests/unit/test_pack_management.py +++ b/st2common/tests/unit/test_pack_management.py @@ -29,7 +29,7 @@ from st2common.util.monkey_patch import use_select_poll_workaround use_select_poll_workaround() -from pack_mgmt.download import DownloadGitRepoAction +from st2common.util.pack_management import eval_repo_url __all__ = [ 'InstallPackTestCase' @@ -39,16 +39,16 @@ class InstallPackTestCase(unittest2.TestCase): def test_eval_repo(self): - result = DownloadGitRepoAction._eval_repo_url('stackstorm/st2contrib') + result = eval_repo_url('stackstorm/st2contrib') self.assertEqual(result, 'https://github.com/stackstorm/st2contrib') - result = DownloadGitRepoAction._eval_repo_url('git@github.com:StackStorm/st2contrib.git') + result = eval_repo_url('git@github.com:StackStorm/st2contrib.git') self.assertEqual(result, 'git@github.com:StackStorm/st2contrib.git') repo_url = 'https://github.com/StackStorm/st2contrib.git' - result = DownloadGitRepoAction._eval_repo_url(repo_url) + result = eval_repo_url(repo_url) self.assertEqual(result, repo_url) repo_url = 'https://git-wip-us.apache.org/repos/asf/libcloud.git' - result = DownloadGitRepoAction._eval_repo_url(repo_url) + result = eval_repo_url(repo_url) self.assertEqual(result, repo_url) diff --git a/st2reactor/st2reactor/cmd/rule_tester.py b/st2reactor/st2reactor/cmd/rule_tester.py index 7354f3ecab..6ba466ea31 100644 --- a/st2reactor/st2reactor/cmd/rule_tester.py +++ b/st2reactor/st2reactor/cmd/rule_tester.py @@ -20,6 +20,7 @@ from st2common import config from st2common import log as logging +from st2common.config import do_register_cli_opts from st2common.script_setup import setup as common_setup from st2common.script_setup import teardown as common_teardown from st2reactor.rules.tester import RuleTester @@ -31,15 +32,6 @@ LOG = logging.getLogger(__name__) -def _do_register_cli_opts(opts, ignore_errors=False): - for opt in opts: - try: - cfg.CONF.register_cli_opt(opt) - except: - if not ignore_errors: - raise - - def _register_cli_opts(): cli_opts = [ cfg.StrOpt('rule', default=None, @@ -51,7 +43,7 @@ def _register_cli_opts(): cfg.StrOpt('trigger-instance-id', default=None, help='Id of the Trigger Instance to use for validation.') ] - _do_register_cli_opts(cli_opts) + do_register_cli_opts(cli_opts) def main(): diff --git a/st2reactor/st2reactor/container/process_container.py b/st2reactor/st2reactor/container/process_container.py index 33aaec95cf..1c9d0dc596 100644 --- a/st2reactor/st2reactor/container/process_container.py +++ b/st2reactor/st2reactor/container/process_container.py @@ -183,9 +183,10 @@ def _poll_sensors_for_results(self, sensor_ids): else: sensor_start_time = self._sensor_start_times[sensor_id] sensor_respawn_count = self._sensor_respawn_counts[sensor_id] - successfuly_started = (now - sensor_start_time) >= SENSOR_SUCCESSFUL_START_THRESHOLD + successfully_started = ((now - sensor_start_time) >= + SENSOR_SUCCESSFUL_START_THRESHOLD) - if successfuly_started and sensor_respawn_count >= 1: + if successfully_started and sensor_respawn_count >= 1: # Sensor has been successfully running more than threshold seconds, clear the # respawn counter so we can try to restart the sensor if it dies later on self._sensor_respawn_counts[sensor_id] = 0 @@ -402,7 +403,7 @@ def _respawn_sensor(self, sensor_id, sensor, exit_code): if self._single_sensor_mode: # In single sensor mode we want to exit immediately on failure LOG.info('Not respawning a sensor since running in single sensor mode', - extra=extra) + extra=extra) self._stopped = True self._exit_code = exit_code