diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d0460f4ea2..dd4a521b03 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,9 @@ Added * Added st2-auth-ldap pip requirements for LDAP auth integartion. (new feature) #5082 Contributed by @hnanchahal +* Added --register-recreate-virtualenvs flag to st2ctl reload to recreate virtualenvs from scratch. (part of upgrade instructions) [#5167] + Contributed by @winem and @blag + Changed ~~~~~~~ diff --git a/contrib/core/requirements-tests.txt b/contrib/core/requirements-tests.txt index 1dfe969c80..de07eb962f 100644 --- a/contrib/core/requirements-tests.txt +++ b/contrib/core/requirements-tests.txt @@ -1 +1 @@ -mail-parser>=3.9.1,<4.0 +mail-parser>=3.9.1,<3.10.0 diff --git a/st2common/bin/st2ctl b/st2common/bin/st2ctl index f2cd03ee22..9584efb4a7 100755 --- a/st2common/bin/st2ctl +++ b/st2common/bin/st2ctl @@ -43,19 +43,20 @@ function print_usage() { echo echo "Usage: st2ctl {reload, clean}" echo "optional arguments:" - echo " --register-all Register all." - echo " --register-triggers Register all triggers." - echo " --register-sensors Register all sensors." - echo " --register-rules Register all rules." - echo " --register-runners Register all runners." - echo " --register-actions Register all actions." - echo " --register-aliases Register all aliases." - echo " --register-policies Register all policies." - echo " --register-configs Register all configuration files." - echo " --register-setup-virtualenvs Create Python virtual environments for all the registered packs." - echo " --register-fail-on-failure Exit with non-zero if some resource registration fails. Deprecated. This is now a default behavior." - echo " --register-no-fail-on-failure Don't exit with non-zero if some resource registration fails." - echo " --verbose Output additional debug and informational messages." + echo " --register-all Register all." + echo " --register-triggers Register all triggers." + echo " --register-sensors Register all sensors." + echo " --register-rules Register all rules." + echo " --register-runners Register all runners." + echo " --register-actions Register all actions." + echo " --register-aliases Register all aliases." + echo " --register-policies Register all policies." + echo " --register-configs Register all configuration files." + echo " --register-setup-virtualenvs Create Python virtual environments for all the registered packs." + echo " --register-recreate-virtualenvs (Delete and re-)create Python virtual environments for all the registered packs." + echo " --register-fail-on-failure Exit with non-zero if some resource registration fails. Deprecated. This is now a default behavior." + echo " --register-no-fail-on-failure Don't exit with non-zero if some resource registration fails." + echo " --verbose Output additional debug and informational messages." echo "" echo "Most commands require elevated privileges." } @@ -140,10 +141,10 @@ function reopen_component_log_files() { } function register_content() { - ALLOWED_REGISTER_FLAGS='--register-all --register-actions --register-aliases --register-runners --register-policies --register-rules --register-sensors --register-triggers --register-configs --register-setup-virtualenvs --register-fail-on-failure --register-no-fail-on-failure --verbose' + ALLOWED_REGISTER_FLAGS='--register-all --register-actions --register-aliases --register-runners --register-policies --register-rules --register-sensors --register-triggers --register-configs --register-setup-virtualenvs --register-recreate-virtualenvs --register-fail-on-failure --register-no-fail-on-failure --verbose' DEFAULT_REGISTER_FLAGS='--register-runners --register-actions --register-aliases --register-sensors --register-triggers --register-configs --register-rules' - SUDO_FLAGS='--register-setup-virtualenvs' + SUDO_FLAGS='--register-setup-virtualenvs --register-recreate-virtualenvs' flags="${@}" if [ ! -z ${1} ]; then diff --git a/st2common/st2common/content/bootstrap.py b/st2common/st2common/content/bootstrap.py index 1072d35053..7690e2dd01 100644 --- a/st2common/st2common/content/bootstrap.py +++ b/st2common/st2common/content/bootstrap.py @@ -63,6 +63,9 @@ def register_opts(): cfg.StrOpt('runner-dir', default=None, help='Directory to load runners from.'), cfg.BoolOpt('setup-virtualenvs', default=False, help=('Setup Python virtual environments ' 'all the Python runner actions.')), + cfg.BoolOpt('recreate-virtualenvs', default=False, help=('Recreate Python virtual ' + 'environments for all the Python ' + 'Python runner actions.')), # General options # Note: This value should default to False since we want fail on failure behavior by @@ -83,7 +86,7 @@ def register_opts(): register_opts() -def setup_virtualenvs(): +def setup_virtualenvs(recreate_virtualenvs=False): """ Setup Python virtual environments for all the registered or the provided pack. """ @@ -110,10 +113,25 @@ def setup_virtualenvs(): # 2. Retrieve available packs (aka packs which have been registered) pack_names = registrar.get_registered_packs() + if recreate_virtualenvs: + """ + update = False: + this is more than an update of an existing virtualenv + the virtualenv itself will be removed & recreated + this is i.e. useful for updates to a newer Python release + """ + update = False + else: + """ + update = True: + only dependencies inside the virtualenv will be updated + """ + update = True + setup_count = 0 for pack_name in pack_names: try: - setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG) + setup_pack_virtualenv(pack_name=pack_name, update=update, logger=LOG) except Exception as e: exc_info = not fail_on_failure LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e, @@ -393,6 +411,9 @@ def register_content(): if cfg.CONF.register.setup_virtualenvs: setup_virtualenvs() + if cfg.CONF.register.recreate_virtualenvs: + setup_virtualenvs(recreate_virtualenvs=True) + def setup(argv): common_setup(config=config, setup_db=True, register_mq_exchanges=True, diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py index db56e6fb20..52e39974fd 100644 --- a/st2common/st2common/util/virtualenvs.py +++ b/st2common/st2common/util/virtualenvs.py @@ -203,6 +203,7 @@ def remove_virtualenv(virtualenv_path, logger=None): logger.debug('Removing virtualenv in "%s"' % virtualenv_path) try: shutil.rmtree(virtualenv_path) + logger.debug('Virtualenv successfull removed.') except Exception as e: logger.error('Error while removing virtualenv at "%s": "%s"' % (virtualenv_path, e)) raise e diff --git a/st2common/tests/integration/test_register_content_script.py b/st2common/tests/integration/test_register_content_script.py index 1d7ca955f9..57568df1c2 100644 --- a/st2common/tests/integration/test_register_content_script.py +++ b/st2common/tests/integration/test_register_content_script.py @@ -155,3 +155,16 @@ def test_register_setup_virtualenvs(self): self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr) self.assertIn('Setup virtualenv for 1 pack(s)', stderr) self.assertEqual(exit_code, 0) + + def test_register_recreate_virtualenvs(self): + # Single pack + pack_dir = os.path.join(get_fixtures_packs_base_path(), 'dummy_pack_1') + + cmd = BASE_CMD_ARGS + ['--register-pack=%s' % (pack_dir), '--register-recreate-virtualenvs', + '--register-no-fail-on-failure'] + exit_code, stdout, stderr = run_command(cmd=cmd) + + self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr) + self.assertIn('Setup virtualenv for 1 pack(s)', stderr) + self.assertIn('Virtualenv successfull removed.', stderr) + self.assertEqual(exit_code, 0)