Add support to force recreation of packs virtualenvs

CHANGELOG.rst

@@ -18,6 +18,9 @@ Added
 * Added st2-auth-ldap pip requirements for LDAP auth integartion. (new feature) #5082
   Contributed by @hnanchahal
 
+* Added --register-recreate-virtualenvs flag to st2ctl reload to recreate virtualenvs from scratch. (part of upgrade instructions) [#5167]
+  Contributed by @winem and @blag
+
 Changed
 ~~~~~~~
 

contrib/core/requirements-tests.txt

@@ -1 +1 @@
-mail-parser>=3.9.1,<4.0
+mail-parser>=3.9.1,<3.10.0

st2common/bin/st2ctl

@@ -43,19 +43,20 @@ function print_usage() {
     echo
     echo "Usage: st2ctl {reload, clean}"
     echo "optional arguments:"
-    echo "  --register-all                Register all."
-    echo "  --register-triggers           Register all triggers."
-    echo "  --register-sensors            Register all sensors."
-    echo "  --register-rules              Register all rules."
-    echo "  --register-runners            Register all runners."
-    echo "  --register-actions            Register all actions."
-    echo "  --register-aliases            Register all aliases."
-    echo "  --register-policies           Register all policies."
-    echo "  --register-configs            Register all configuration files."
-    echo "  --register-setup-virtualenvs  Create Python virtual environments for all the registered packs."
-    echo "  --register-fail-on-failure    Exit with non-zero if some resource registration fails. Deprecated. This is now a default behavior."
-    echo "  --register-no-fail-on-failure Don't exit with non-zero if some resource registration fails."
-    echo "  --verbose                     Output additional debug and informational messages."
+    echo "  --register-all                  Register all."
+    echo "  --register-triggers             Register all triggers."
+    echo "  --register-sensors              Register all sensors."
+    echo "  --register-rules                Register all rules."
+    echo "  --register-runners              Register all runners."
+    echo "  --register-actions              Register all actions."
+    echo "  --register-aliases              Register all aliases."
+    echo "  --register-policies             Register all policies."
+    echo "  --register-configs              Register all configuration files."
+    echo "  --register-setup-virtualenvs    Create Python virtual environments for all the registered packs."
+    echo "  --register-recreate-virtualenvs (Delete and re-)create Python virtual environments for all the registered packs."
+    echo "  --register-fail-on-failure      Exit with non-zero if some resource registration fails. Deprecated. This is now a default behavior."
+    echo "  --register-no-fail-on-failure   Don't exit with non-zero if some resource registration fails."
+    echo "  --verbose                       Output additional debug and informational messages."
     echo ""
     echo "Most commands require elevated privileges."
 }
@@ -140,10 +141,10 @@ function reopen_component_log_files() {
 }
 
 function register_content() {
-  ALLOWED_REGISTER_FLAGS='--register-all --register-actions --register-aliases --register-runners --register-policies --register-rules --register-sensors --register-triggers --register-configs --register-setup-virtualenvs --register-fail-on-failure --register-no-fail-on-failure --verbose'
+  ALLOWED_REGISTER_FLAGS='--register-all --register-actions --register-aliases --register-runners --register-policies --register-rules --register-sensors --register-triggers --register-configs --register-setup-virtualenvs --register-recreate-virtualenvs --register-fail-on-failure --register-no-fail-on-failure --verbose'
   DEFAULT_REGISTER_FLAGS='--register-runners --register-actions --register-aliases --register-sensors --register-triggers --register-configs --register-rules'
 
-  SUDO_FLAGS='--register-setup-virtualenvs'
+  SUDO_FLAGS='--register-setup-virtualenvs  --register-recreate-virtualenvs'
   flags="${@}"
 
   if [ ! -z ${1} ]; then

st2common/st2common/content/bootstrap.py

@@ -63,6 +63,9 @@ def register_opts():
         cfg.StrOpt('runner-dir', default=None, help='Directory to load runners from.'),
         cfg.BoolOpt('setup-virtualenvs', default=False, help=('Setup Python virtual environments '
                                                               'all the Python runner actions.')),
+        cfg.BoolOpt('recreate-virtualenvs', default=False, help=('Recreate Python virtual '
+                                                                 'environments for all the Python '
+                                                                 'Python runner actions.')),
 
         # General options
         # Note: This value should default to False since we want fail on failure behavior by
@@ -83,7 +86,7 @@ def register_opts():
 register_opts()
 
 
-def setup_virtualenvs():
+def setup_virtualenvs(recreate_virtualenvs=False):
     """
     Setup Python virtual environments for all the registered or the provided pack.
     """
@@ -110,10 +113,25 @@ def setup_virtualenvs():
         # 2. Retrieve available packs (aka packs which have been registered)
         pack_names = registrar.get_registered_packs()
 
+    if recreate_virtualenvs:
+        """
+        update = False:
+        this is more than an update of an existing virtualenv
+        the virtualenv itself will be removed & recreated
+        this is i.e. useful for updates to a newer Python release
+        """
+        update = False
+    else:
+        """
+        update = True:
+        only dependencies inside the virtualenv will be updated
+        """
+        update = True
+
     setup_count = 0
     for pack_name in pack_names:
         try:
-            setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG)
+            setup_pack_virtualenv(pack_name=pack_name, update=update, logger=LOG)
         except Exception as e:
             exc_info = not fail_on_failure
             LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e,
@@ -393,6 +411,9 @@ def register_content():
     if cfg.CONF.register.setup_virtualenvs:
         setup_virtualenvs()
 
+    if cfg.CONF.register.recreate_virtualenvs:
+        setup_virtualenvs(recreate_virtualenvs=True)
+
 
 def setup(argv):
     common_setup(config=config, setup_db=True, register_mq_exchanges=True,

st2common/st2common/util/virtualenvs.py

@@ -203,6 +203,7 @@ def remove_virtualenv(virtualenv_path, logger=None):
     logger.debug('Removing virtualenv in "%s"' % virtualenv_path)
     try:
         shutil.rmtree(virtualenv_path)
+        logger.debug('Virtualenv successfull removed.')
     except Exception as e:
         logger.error('Error while removing virtualenv at "%s": "%s"' % (virtualenv_path, e))
         raise e

st2common/tests/integration/test_register_content_script.py

@@ -155,3 +155,16 @@ def test_register_setup_virtualenvs(self):
         self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
         self.assertIn('Setup virtualenv for 1 pack(s)', stderr)
         self.assertEqual(exit_code, 0)
+
+    def test_register_recreate_virtualenvs(self):
+        # Single pack
+        pack_dir = os.path.join(get_fixtures_packs_base_path(), 'dummy_pack_1')
+
+        cmd = BASE_CMD_ARGS + ['--register-pack=%s' % (pack_dir), '--register-recreate-virtualenvs',
+                               '--register-no-fail-on-failure']
+        exit_code, stdout, stderr = run_command(cmd=cmd)
+
+        self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
+        self.assertIn('Setup virtualenv for 1 pack(s)', stderr)
+        self.assertIn('Virtualenv successfull removed.', stderr)
+        self.assertEqual(exit_code, 0)