From 1ea7c459fa9819832944a4e6dd3fa6686965587f Mon Sep 17 00:00:00 2001 From: Lakshmi Kannan Date: Mon, 11 Jan 2016 17:57:29 -0800 Subject: [PATCH 1/5] Add --always-copy option to create_virtualenv method for pack virtualenvs --- Makefile | 1 + contrib/packs/actions/pack_mgmt/setup_virtualenv.py | 13 +++++++++++-- st2actions/st2actions/config.py | 13 +++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0c9d9e0134..b70ac57a9d 100644 --- a/Makefile +++ b/Makefile @@ -167,6 +167,7 @@ requirements: virtualenv .sdist-requirements # Make sure we use latest version of pip $(VIRTUALENV_DIR)/bin/pip install --upgrade pip + $(VIRTUALENV_DIR)/bin/pip install virtualenv # Required for packs.install in dev envs. # Generate all requirements to support current CI pipeline. $(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv -s st2*/in-requirements.txt -f fixed-requirements.txt -o requirements.txt diff --git a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py index 67cb84ddb8..2f5dcbba9b 100644 --- a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py +++ b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py @@ -125,6 +125,8 @@ def _setup_pack_virtualenv(self, pack_name, update=False): def _create_virtualenv(self, virtualenv_path): python_binary = cfg.CONF.actionrunner.python_binary + virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary + virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts if not os.path.isfile(python_binary): raise Exception('Python binary "%s" doesn\'t exist' % (python_binary)) @@ -132,8 +134,15 @@ def _create_virtualenv(self, virtualenv_path): self.logger.debug('Creating virtualenv in "%s" using Python binary "%s"' % (virtualenv_path, python_binary)) - cmd = ['virtualenv', '-p', python_binary, '--system-site-packages', virtualenv_path] - exit_code, _, stderr = run_command(cmd=cmd) + cmd = [virtualenv_binary, '-p', python_binary] + cmd.extend(virtualenv_opts) + cmd.extend([virtualenv_path]) + self.logger.debug('Running command "%s" to create virtualenv.', ' '.join(cmd)) + + try: + exit_code, _, stderr = run_command(cmd=cmd) + except OSError: + raise Exception('Virtualenv binary "%s" doesn\'t exist.' % virtualenv_binary) if exit_code != 0: raise Exception('Failed to create virtualenv in "%s": %s' % diff --git a/st2actions/st2actions/config.py b/st2actions/st2actions/config.py index 483635ae2b..14b6d52366 100644 --- a/st2actions/st2actions/config.py +++ b/st2actions/st2actions/config.py @@ -17,6 +17,7 @@ Configuration options registration and useful routines. """ +import os import sys from oslo_config import cfg @@ -41,11 +42,19 @@ def _register_common_opts(): def _register_action_runner_opts(): + default_python_bin_path = sys.executable + base_dir = os.path.dirname(os.path.realpath(default_python_bin_path)) + default_virtualenv_bin_path = os.path.join(base_dir, 'virtualenv') logging_opts = [ cfg.StrOpt('logging', default='conf/logging.conf', help='location of the logging.conf file'), - cfg.StrOpt('python_binary', default=sys.executable, - help='Python binary which will be used by Python actions.') + cfg.StrOpt('python_binary', default=default_python_bin_path, + help='Python binary which will be used by Python actions.'), + cfg.StrOpt('virtualenv_binary', default=default_virtualenv_bin_path, + help='Virtualenv binary which should be used to create pack virtualenvs.'), + cfg.ListOpt('virtualenv_opts', default=['--always-copy', '--system-site-packages'], + help='List of virtualenv options to be passsed to "virtualenv" command that ' + + 'creates pack virtualenv.') ] CONF.register_opts(logging_opts, group='actionrunner') From b938ff0635ce403c58077d15ab9dd096de993c95 Mon Sep 17 00:00:00 2001 From: Lakshmi Kannan Date: Tue, 12 Jan 2016 16:23:39 -0800 Subject: [PATCH 2/5] Better error messages for OSError --- contrib/packs/actions/pack_mgmt/setup_virtualenv.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py index 2f5dcbba9b..d813dbf673 100644 --- a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py +++ b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py @@ -131,6 +131,9 @@ def _create_virtualenv(self, virtualenv_path): if not os.path.isfile(python_binary): raise Exception('Python binary "%s" doesn\'t exist' % (python_binary)) + if not os.path.isfile(virtualenv_binary): + raise Exception('Virtualenv binary "%s" doesn\'t exist.' % (virtualenv_binary)) + self.logger.debug('Creating virtualenv in "%s" using Python binary "%s"' % (virtualenv_path, python_binary)) @@ -141,8 +144,9 @@ def _create_virtualenv(self, virtualenv_path): try: exit_code, _, stderr = run_command(cmd=cmd) - except OSError: - raise Exception('Virtualenv binary "%s" doesn\'t exist.' % virtualenv_binary) + except OSError as e: + raise Exception('Error executing command %s. %s.' % (' '.join(cmd), + e.message)) if exit_code != 0: raise Exception('Failed to create virtualenv in "%s": %s' % From 20fe3cb9ddf5d9b399c787681e44a9dc89efaecf Mon Sep 17 00:00:00 2001 From: Lakshmi Kannan Date: Tue, 12 Jan 2016 17:49:18 -0800 Subject: [PATCH 3/5] Update CHANGELOG --- CHANGELOG.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cc75a4028d..1e0aabc538 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -44,7 +44,9 @@ in development * Fix action parameters validation so that only a selected set of attributes can be overriden for any runner parameters. (bug fix) * Fix type in the headers parameter for the http-request runner. (bug fix) -* Fix runaway action triggers caused by state miscalculation for mistral workflow. (bug fix) +* Fix runaway action triggers caused by state miscalculation for mistral workflow. (bug fix) +* Use ``--always-copy`` option when creating virtualenv for packs from packs.setup_virtualenv action. This is required when st2actionrunner is kicked off +from python within a virtualenv. 1.2.0 - December 07, 2015 ------------------------- From ae800517808b5cd6e949107c91d34b3405b8179a Mon Sep 17 00:00:00 2001 From: Lakshmi Kannan Date: Wed, 13 Jan 2016 10:26:41 -0800 Subject: [PATCH 4/5] Set virtualenv_opts to --always-copy in st2.conf for new packages --- conf/st2.package.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/st2.package.conf b/conf/st2.package.conf index e8042c26cb..f1ae43c8a0 100644 --- a/conf/st2.package.conf +++ b/conf/st2.package.conf @@ -17,6 +17,7 @@ logging = /etc/st2/logging.rulesengine.conf [actionrunner] logging = /etc/st2/logging.actionrunner.conf +virtualenv_opts = --always-copy [resultstracker] logging = /etc/st2/logging.resultstracker.conf From 8f421dffb4825e0832acce9b0440046d309f6795 Mon Sep 17 00:00:00 2001 From: Lakshmi Kannan Date: Wed, 13 Jan 2016 10:27:08 -0800 Subject: [PATCH 5/5] Comma separated list is the only one supported for ListOpt in oslo --- conf/st2.conf.sample | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index 211b28bce7..6400d80b59 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -13,7 +13,7 @@ logging = conf/logging.conf [api] # List of origins allowed -allow_origin = ['http://localhost:3000'] +allow_origin = http://localhost:3000 # comma separated list allowed here. # location of the logging.conf file logging = conf/logging.conf # True to mask secrets in API responses @@ -114,7 +114,7 @@ collection_interval = 600 # Controls if stderr should be redirected to the logs. redirect_stderr = False # Exclusion list of loggers to omit. -excludes = +excludes = # True to mask secrets in the log files. mask_secrets = True @@ -122,7 +122,7 @@ mask_secrets = True # URL of the messaging server. url = amqp://guest:guest@127.0.0.1:5672// # URL of all the nodes in a messaging service cluster. -cluster_urls = [] +cluster_urls = # comma separated list or URLs allowed here. [mistral] # URL Mistral uses to talk back to the API.If not provided it defaults to public API URL. Note: This needs to be a base URL without API version (e.g. http://127.0.0.1:9101)