From 70ba17326a66fc7ff225d4ce230005f69037b7f0 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 25 Feb 2021 22:37:18 +0100 Subject: [PATCH 1/9] Add support to force recreation of packs virtualenvs --- st2common/bin/st2ctl | 29 ++++++++++++------------ st2common/st2common/content/bootstrap.py | 21 +++++++++++++++-- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/st2common/bin/st2ctl b/st2common/bin/st2ctl index f2cd03ee22..a33d427c7c 100755 --- a/st2common/bin/st2ctl +++ b/st2common/bin/st2ctl @@ -43,19 +43,20 @@ function print_usage() { echo echo "Usage: st2ctl {reload, clean}" echo "optional arguments:" - echo " --register-all Register all." - echo " --register-triggers Register all triggers." - echo " --register-sensors Register all sensors." - echo " --register-rules Register all rules." - echo " --register-runners Register all runners." - echo " --register-actions Register all actions." - echo " --register-aliases Register all aliases." - echo " --register-policies Register all policies." - echo " --register-configs Register all configuration files." - echo " --register-setup-virtualenvs Create Python virtual environments for all the registered packs." - echo " --register-fail-on-failure Exit with non-zero if some resource registration fails. Deprecated. This is now a default behavior." - echo " --register-no-fail-on-failure Don't exit with non-zero if some resource registration fails." - echo " --verbose Output additional debug and informational messages." + echo " --register-all Register all." + echo " --register-triggers Register all triggers." + echo " --register-sensors Register all sensors." + echo " --register-rules Register all rules." + echo " --register-runners Register all runners." + echo " --register-actions Register all actions." + echo " --register-aliases Register all aliases." + echo " --register-policies Register all policies." + echo " --register-configs Register all configuration files." + echo " --register-setup-virtualenvs Create Python virtual environments for all the registered packs." + echo " --register-setup-virtualenvs-force-recreation (Delete and re-)create Python virtual environments for all the registered packs." + echo " --register-fail-on-failure Exit with non-zero if some resource registration fails. Deprecated. This is now a default behavior." + echo " --register-no-fail-on-failure Don't exit with non-zero if some resource registration fails." + echo " --verbose Output additional debug and informational messages." echo "" echo "Most commands require elevated privileges." } @@ -140,7 +141,7 @@ function reopen_component_log_files() { } function register_content() { - ALLOWED_REGISTER_FLAGS='--register-all --register-actions --register-aliases --register-runners --register-policies --register-rules --register-sensors --register-triggers --register-configs --register-setup-virtualenvs --register-fail-on-failure --register-no-fail-on-failure --verbose' + ALLOWED_REGISTER_FLAGS='--register-all --register-actions --register-aliases --register-runners --register-policies --register-rules --register-sensors --register-triggers --register-configs --register-setup-virtualenvs --register-setup-virtualenvs-force-recreation --register-fail-on-failure --register-no-fail-on-failure --verbose' DEFAULT_REGISTER_FLAGS='--register-runners --register-actions --register-aliases --register-sensors --register-triggers --register-configs --register-rules' SUDO_FLAGS='--register-setup-virtualenvs' diff --git a/st2common/st2common/content/bootstrap.py b/st2common/st2common/content/bootstrap.py index 1072d35053..5725027022 100644 --- a/st2common/st2common/content/bootstrap.py +++ b/st2common/st2common/content/bootstrap.py @@ -83,7 +83,7 @@ def register_opts(): register_opts() -def setup_virtualenvs(): +def setup_virtualenvs(recreate_virtualenvs=False): """ Setup Python virtual environments for all the registered or the provided pack. """ @@ -110,10 +110,24 @@ def setup_virtualenvs(): # 2. Retrieve available packs (aka packs which have been registered) pack_names = registrar.get_registered_packs() + if recreate_virtualenvs: + """ + update = False: + this is more than an update of an existing virtualenv + the virtualenv itself will be removed & recreated to apply i.e. updates to a newer Python release + """ + update = False + else: + """ + update = True: + only dependencies inside the virtualenv will be updated + """ + update = True + setup_count = 0 for pack_name in pack_names: try: - setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG) + setup_pack_virtualenv(pack_name=pack_name, update=update, logger=LOG) except Exception as e: exc_info = not fail_on_failure LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e, @@ -393,6 +407,9 @@ def register_content(): if cfg.CONF.register.setup_virtualenvs: setup_virtualenvs() + if cfg.CONF.register.setup_virtualenvs_force_recreation: + setup_virtualenvs(update=False) + def setup(argv): common_setup(config=config, setup_db=True, register_mq_exchanges=True, From b09206958f01e5db6777d59557cf9a267b6b414d Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 25 Feb 2021 22:59:17 +0100 Subject: [PATCH 2/9] Update setup_virtualenvs parameter to the new name --- st2common/st2common/content/bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/content/bootstrap.py b/st2common/st2common/content/bootstrap.py index 5725027022..19a81a1a6e 100644 --- a/st2common/st2common/content/bootstrap.py +++ b/st2common/st2common/content/bootstrap.py @@ -408,7 +408,7 @@ def register_content(): setup_virtualenvs() if cfg.CONF.register.setup_virtualenvs_force_recreation: - setup_virtualenvs(update=False) + setup_virtualenvs(recreate_virtualenvs=False) def setup(argv): From 4f0e3ad3dfbd63ee779bf0fe18173ea0222bbfc7 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Thu, 25 Feb 2021 23:05:19 +0100 Subject: [PATCH 3/9] Rename --register-setup-virtualenvs-force-recreation to --register-setup-recreate-virtualenvs --- st2common/bin/st2ctl | 30 ++++++++++++------------ st2common/st2common/content/bootstrap.py | 4 ++-- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/st2common/bin/st2ctl b/st2common/bin/st2ctl index a33d427c7c..b39d8fc8f1 100755 --- a/st2common/bin/st2ctl +++ b/st2common/bin/st2ctl @@ -43,20 +43,20 @@ function print_usage() { echo echo "Usage: st2ctl {reload, clean}" echo "optional arguments:" - echo " --register-all Register all." - echo " --register-triggers Register all triggers." - echo " --register-sensors Register all sensors." - echo " --register-rules Register all rules." - echo " --register-runners Register all runners." - echo " --register-actions Register all actions." - echo " --register-aliases Register all aliases." - echo " --register-policies Register all policies." - echo " --register-configs Register all configuration files." - echo " --register-setup-virtualenvs Create Python virtual environments for all the registered packs." - echo " --register-setup-virtualenvs-force-recreation (Delete and re-)create Python virtual environments for all the registered packs." - echo " --register-fail-on-failure Exit with non-zero if some resource registration fails. Deprecated. This is now a default behavior." - echo " --register-no-fail-on-failure Don't exit with non-zero if some resource registration fails." - echo " --verbose Output additional debug and informational messages." + echo " --register-all Register all." + echo " --register-triggers Register all triggers." + echo " --register-sensors Register all sensors." + echo " --register-rules Register all rules." + echo " --register-runners Register all runners." + echo " --register-actions Register all actions." + echo " --register-aliases Register all aliases." + echo " --register-policies Register all policies." + echo " --register-configs Register all configuration files." + echo " --register-setup-virtualenvs Create Python virtual environments for all the registered packs." + echo " --register-setup-recreate-virtualenvs (Delete and re-)create Python virtual environments for all the registered packs." + echo " --register-fail-on-failure Exit with non-zero if some resource registration fails. Deprecated. This is now a default behavior." + echo " --register-no-fail-on-failure Don't exit with non-zero if some resource registration fails." + echo " --verbose Output additional debug and informational messages." echo "" echo "Most commands require elevated privileges." } @@ -141,7 +141,7 @@ function reopen_component_log_files() { } function register_content() { - ALLOWED_REGISTER_FLAGS='--register-all --register-actions --register-aliases --register-runners --register-policies --register-rules --register-sensors --register-triggers --register-configs --register-setup-virtualenvs --register-setup-virtualenvs-force-recreation --register-fail-on-failure --register-no-fail-on-failure --verbose' + ALLOWED_REGISTER_FLAGS='--register-all --register-actions --register-aliases --register-runners --register-policies --register-rules --register-sensors --register-triggers --register-configs --register-setup-virtualenvs --register-setup-recreate-virtualenvs --register-fail-on-failure --register-no-fail-on-failure --verbose' DEFAULT_REGISTER_FLAGS='--register-runners --register-actions --register-aliases --register-sensors --register-triggers --register-configs --register-rules' SUDO_FLAGS='--register-setup-virtualenvs' diff --git a/st2common/st2common/content/bootstrap.py b/st2common/st2common/content/bootstrap.py index 19a81a1a6e..ce0a803310 100644 --- a/st2common/st2common/content/bootstrap.py +++ b/st2common/st2common/content/bootstrap.py @@ -407,8 +407,8 @@ def register_content(): if cfg.CONF.register.setup_virtualenvs: setup_virtualenvs() - if cfg.CONF.register.setup_virtualenvs_force_recreation: - setup_virtualenvs(recreate_virtualenvs=False) + if cfg.CONF.register.setup_recreate_virtualenvs: + setup_virtualenvs(recreate_virtualenvs=True) def setup(argv): From 60d37b580582efcc9f12831cafb716372216e540 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 26 Feb 2021 21:15:43 +0100 Subject: [PATCH 4/9] Switch to shorter register-recreate-virtualenvs and add the config option --- st2common/bin/st2ctl | 32 ++++++++++++------------ st2common/st2common/content/bootstrap.py | 4 ++- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/st2common/bin/st2ctl b/st2common/bin/st2ctl index b39d8fc8f1..9584efb4a7 100755 --- a/st2common/bin/st2ctl +++ b/st2common/bin/st2ctl @@ -43,20 +43,20 @@ function print_usage() { echo echo "Usage: st2ctl {reload, clean}" echo "optional arguments:" - echo " --register-all Register all." - echo " --register-triggers Register all triggers." - echo " --register-sensors Register all sensors." - echo " --register-rules Register all rules." - echo " --register-runners Register all runners." - echo " --register-actions Register all actions." - echo " --register-aliases Register all aliases." - echo " --register-policies Register all policies." - echo " --register-configs Register all configuration files." - echo " --register-setup-virtualenvs Create Python virtual environments for all the registered packs." - echo " --register-setup-recreate-virtualenvs (Delete and re-)create Python virtual environments for all the registered packs." - echo " --register-fail-on-failure Exit with non-zero if some resource registration fails. Deprecated. This is now a default behavior." - echo " --register-no-fail-on-failure Don't exit with non-zero if some resource registration fails." - echo " --verbose Output additional debug and informational messages." + echo " --register-all Register all." + echo " --register-triggers Register all triggers." + echo " --register-sensors Register all sensors." + echo " --register-rules Register all rules." + echo " --register-runners Register all runners." + echo " --register-actions Register all actions." + echo " --register-aliases Register all aliases." + echo " --register-policies Register all policies." + echo " --register-configs Register all configuration files." + echo " --register-setup-virtualenvs Create Python virtual environments for all the registered packs." + echo " --register-recreate-virtualenvs (Delete and re-)create Python virtual environments for all the registered packs." + echo " --register-fail-on-failure Exit with non-zero if some resource registration fails. Deprecated. This is now a default behavior." + echo " --register-no-fail-on-failure Don't exit with non-zero if some resource registration fails." + echo " --verbose Output additional debug and informational messages." echo "" echo "Most commands require elevated privileges." } @@ -141,10 +141,10 @@ function reopen_component_log_files() { } function register_content() { - ALLOWED_REGISTER_FLAGS='--register-all --register-actions --register-aliases --register-runners --register-policies --register-rules --register-sensors --register-triggers --register-configs --register-setup-virtualenvs --register-setup-recreate-virtualenvs --register-fail-on-failure --register-no-fail-on-failure --verbose' + ALLOWED_REGISTER_FLAGS='--register-all --register-actions --register-aliases --register-runners --register-policies --register-rules --register-sensors --register-triggers --register-configs --register-setup-virtualenvs --register-recreate-virtualenvs --register-fail-on-failure --register-no-fail-on-failure --verbose' DEFAULT_REGISTER_FLAGS='--register-runners --register-actions --register-aliases --register-sensors --register-triggers --register-configs --register-rules' - SUDO_FLAGS='--register-setup-virtualenvs' + SUDO_FLAGS='--register-setup-virtualenvs --register-recreate-virtualenvs' flags="${@}" if [ ! -z ${1} ]; then diff --git a/st2common/st2common/content/bootstrap.py b/st2common/st2common/content/bootstrap.py index ce0a803310..1ddabbc8d5 100644 --- a/st2common/st2common/content/bootstrap.py +++ b/st2common/st2common/content/bootstrap.py @@ -63,6 +63,8 @@ def register_opts(): cfg.StrOpt('runner-dir', default=None, help='Directory to load runners from.'), cfg.BoolOpt('setup-virtualenvs', default=False, help=('Setup Python virtual environments ' 'all the Python runner actions.')), + cfg.BoolOpt('recreate-virtualenvs', default=False, help=('Recreate Python virtual environments ' + 'all the Python runner actions.')), # General options # Note: This value should default to False since we want fail on failure behavior by @@ -407,7 +409,7 @@ def register_content(): if cfg.CONF.register.setup_virtualenvs: setup_virtualenvs() - if cfg.CONF.register.setup_recreate_virtualenvs: + if cfg.CONF.register.recreate_virtualenvs: setup_virtualenvs(recreate_virtualenvs=True) From 699117568c946e638e8946059d2748f33c67b3b4 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 26 Feb 2021 21:25:09 +0100 Subject: [PATCH 5/9] shorten comments to be lint compliant --- st2common/st2common/content/bootstrap.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/st2common/st2common/content/bootstrap.py b/st2common/st2common/content/bootstrap.py index 1ddabbc8d5..7690e2dd01 100644 --- a/st2common/st2common/content/bootstrap.py +++ b/st2common/st2common/content/bootstrap.py @@ -63,8 +63,9 @@ def register_opts(): cfg.StrOpt('runner-dir', default=None, help='Directory to load runners from.'), cfg.BoolOpt('setup-virtualenvs', default=False, help=('Setup Python virtual environments ' 'all the Python runner actions.')), - cfg.BoolOpt('recreate-virtualenvs', default=False, help=('Recreate Python virtual environments ' - 'all the Python runner actions.')), + cfg.BoolOpt('recreate-virtualenvs', default=False, help=('Recreate Python virtual ' + 'environments for all the Python ' + 'Python runner actions.')), # General options # Note: This value should default to False since we want fail on failure behavior by @@ -116,7 +117,8 @@ def setup_virtualenvs(recreate_virtualenvs=False): """ update = False: this is more than an update of an existing virtualenv - the virtualenv itself will be removed & recreated to apply i.e. updates to a newer Python release + the virtualenv itself will be removed & recreated + this is i.e. useful for updates to a newer Python release """ update = False else: From 6dae997c333a41754bff50ced1fa9c86955e3cae Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 26 Feb 2021 22:04:29 +0100 Subject: [PATCH 6/9] Add log message and test for register-recreate-virtualenvs --- st2common/st2common/util/virtualenvs.py | 1 + .../integration/test_register_content_script.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py index db56e6fb20..52e39974fd 100644 --- a/st2common/st2common/util/virtualenvs.py +++ b/st2common/st2common/util/virtualenvs.py @@ -203,6 +203,7 @@ def remove_virtualenv(virtualenv_path, logger=None): logger.debug('Removing virtualenv in "%s"' % virtualenv_path) try: shutil.rmtree(virtualenv_path) + logger.debug('Virtualenv successfull removed.') except Exception as e: logger.error('Error while removing virtualenv at "%s": "%s"' % (virtualenv_path, e)) raise e diff --git a/st2common/tests/integration/test_register_content_script.py b/st2common/tests/integration/test_register_content_script.py index 1d7ca955f9..2a14ba44f9 100644 --- a/st2common/tests/integration/test_register_content_script.py +++ b/st2common/tests/integration/test_register_content_script.py @@ -155,3 +155,16 @@ def test_register_setup_virtualenvs(self): self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr) self.assertIn('Setup virtualenv for 1 pack(s)', stderr) self.assertEqual(exit_code, 0) + + def test_register_recreate_virtualenvs(self): + # Single pack + pack_dir = os.path.join(get_fixtures_packs_base_path(), 'dummy_pack_1') + + cmd = BASE_CMD_ARGS + ['--register-pack=%s' % (pack_dir), '--register-recreate-virtualenvs', + '--register-no-fail-on-failure'] + exit_code, stdout, stderr = run_command(cmd=cmd) + + self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr) + self.assertIn('Setup virtualenv for 1 pack(s)', stderr) + self.assertIn('Virtualenv successfull removed.', stderr) + self.assertEqual(exit_code, 0) \ No newline at end of file From e47cb0f8636745d1a3ad24e57e76bd2c6c3f4213 Mon Sep 17 00:00:00 2001 From: Marcel Weinberg Date: Fri, 26 Feb 2021 22:22:10 +0100 Subject: [PATCH 7/9] Add newline at end of file --- st2common/tests/integration/test_register_content_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/tests/integration/test_register_content_script.py b/st2common/tests/integration/test_register_content_script.py index 2a14ba44f9..57568df1c2 100644 --- a/st2common/tests/integration/test_register_content_script.py +++ b/st2common/tests/integration/test_register_content_script.py @@ -167,4 +167,4 @@ def test_register_recreate_virtualenvs(self): self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr) self.assertIn('Setup virtualenv for 1 pack(s)', stderr) self.assertIn('Virtualenv successfull removed.', stderr) - self.assertEqual(exit_code, 0) \ No newline at end of file + self.assertEqual(exit_code, 0) From 36b05fd469b4180ca4551925fe5ec3c5dd2b9100 Mon Sep 17 00:00:00 2001 From: blag Date: Fri, 26 Feb 2021 14:07:39 -0800 Subject: [PATCH 8/9] Pin mail-parser tighter - to 3.9.x --- contrib/core/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/core/requirements-tests.txt b/contrib/core/requirements-tests.txt index 1dfe969c80..de07eb962f 100644 --- a/contrib/core/requirements-tests.txt +++ b/contrib/core/requirements-tests.txt @@ -1 +1 @@ -mail-parser>=3.9.1,<4.0 +mail-parser>=3.9.1,<3.10.0 From d60443dbf74658e55f091f71ac09f8cd0e316654 Mon Sep 17 00:00:00 2001 From: blag Date: Fri, 26 Feb 2021 17:33:16 -0800 Subject: [PATCH 9/9] Add st2ctl changes to changelog --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d0460f4ea2..dd4a521b03 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,9 @@ Added * Added st2-auth-ldap pip requirements for LDAP auth integartion. (new feature) #5082 Contributed by @hnanchahal +* Added --register-recreate-virtualenvs flag to st2ctl reload to recreate virtualenvs from scratch. (part of upgrade instructions) [#5167] + Contributed by @winem and @blag + Changed ~~~~~~~