From d8488b18c0b570fb4e16b0096a272e8bf77dbfbc Mon Sep 17 00:00:00 2001 From: beantaxi Date: Thu, 31 Dec 2020 22:01:36 -0600 Subject: [PATCH 01/49] first commit for first cloud-init PR --- cloudinit/cmd/devel/shell_script_per_boot.py | 26 +++++++++++++++++++ .../cmd/devel/shell_script_per_instance.py | 26 +++++++++++++++++++ cloudinit/cmd/devel/shell_script_per_once.py | 26 +++++++++++++++++++ tools/.github-cla-signers | 1 + 4 files changed, 79 insertions(+) create mode 100644 cloudinit/cmd/devel/shell_script_per_boot.py create mode 100644 cloudinit/cmd/devel/shell_script_per_instance.py create mode 100644 cloudinit/cmd/devel/shell_script_per_once.py diff --git a/cloudinit/cmd/devel/shell_script_per_boot.py b/cloudinit/cmd/devel/shell_script_per_boot.py new file mode 100644 index 00000000000..31b0c982685 --- /dev/null +++ b/cloudinit/cmd/devel/shell_script_per_boot.py @@ -0,0 +1,26 @@ +#part-handler +# vi: syntax=python ts=4 + +from cloudinit import log +from cloudinit import util +import os +import pathlib + +LOG = log.getLogger(__name__) + + +def list_types(): +# return a list of mime-types that are handled by this module + LOG.debug("### in shell_script-per-boot.list_types() ...") + return(["text/x-shellscript-per-boot"]) + + +def handle_part(data, ctype, script_path, payload): + if script_path is not None: + LOG.debug("### in shell_script-per-boot.handle_part() ...") + LOG.debug(f"### x-shellscript-per-boot.handle_part: script_path={script_path}") + (folder, filename) = os.path.split(script_path) + LOG.debug(f"### folder={folder} filename={filename}") + path = f"/var/lib/cloud/scripts/per-boot/{filename}" + LOG.debug(f"### path={path}") + util.write_file(path, payload, 0o700) diff --git a/cloudinit/cmd/devel/shell_script_per_instance.py b/cloudinit/cmd/devel/shell_script_per_instance.py new file mode 100644 index 00000000000..257bbef4d35 --- /dev/null +++ b/cloudinit/cmd/devel/shell_script_per_instance.py @@ -0,0 +1,26 @@ +#part-handler +# vi: syntax=python ts=4 + +from cloudinit import log +from cloudinit import util +import os +import pathlib + +LOG = log.getLogger(__name__) + + +def list_types(): +# return a list of mime-types that are handled by this module + LOG.debug("### in shell_script-per-instance.list_types() ...") + return(["text/x-shellscript-per-instance"]) + + +def handle_part(data, ctype, script_path, payload): + if script_path is not None: + LOG.debug("### in shell_script-per-instance.handle_part() ...") + LOG.debug(f"### x-shellscript-per-instance.handle_part: script_path={script_path}") + (folder, filename) = os.path.split(script_path) + LOG.debug(f"### folder={folder} filename={filename}") + path = f"/var/lib/cloud/scripts/per-instance/{filename}" + LOG.debug(f"### path={path}") + util.write_file(path, payload, 0o700) diff --git a/cloudinit/cmd/devel/shell_script_per_once.py b/cloudinit/cmd/devel/shell_script_per_once.py new file mode 100644 index 00000000000..2a8d06427f8 --- /dev/null +++ b/cloudinit/cmd/devel/shell_script_per_once.py @@ -0,0 +1,26 @@ +part-handler +# vi: syntax=python ts=4 + +from cloudinit import log +from cloudinit import util +import os +import pathlib + +LOG = log.getLogger(__name__) + + +def list_types(): +# return a list of mime-types that are handled by this module + LOG.debug("### in shell_script-per-once.list_types() ...") + return(["text/x-shellscript-per-once"]) + + +def handle_part(data, ctype, script_path, payload): + if script_path is not None: + LOG.debug("### in shell_script-per-once.handle_part() ...") + LOG.debug(f"### x-shellscript-per-once.handle_part: script_path={script_path}") + (folder, filename) = os.path.split(script_path) + LOG.debug(f"### folder={folder} filename={filename}") + path = f"/var/lib/cloud/scripts/per-once/{filename}" + LOG.debug(f"### path={path}") + util.write_file(path, payload, 0o700) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index c843e475071..31fe1f820ab 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -2,6 +2,7 @@ ader1990 AlexBaranowski Aman306 aswinrajamannar +beantaxi beezly bipinbachhao BirknerAlex From bd889659d8a44aa753160ba4aa56491bb9d6b3f3 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Fri, 1 Jan 2021 07:25:52 -0600 Subject: [PATCH 02/49] cleaned up log statements, in part to fix travis --- cloudinit/cmd/devel/shell_script_per_boot.py | 10 +++++----- cloudinit/cmd/devel/shell_script_per_instance.py | 10 +++++----- cloudinit/cmd/devel/shell_script_per_once.py | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cloudinit/cmd/devel/shell_script_per_boot.py b/cloudinit/cmd/devel/shell_script_per_boot.py index 31b0c982685..cf4efa42b59 100644 --- a/cloudinit/cmd/devel/shell_script_per_boot.py +++ b/cloudinit/cmd/devel/shell_script_per_boot.py @@ -11,16 +11,16 @@ def list_types(): # return a list of mime-types that are handled by this module - LOG.debug("### in shell_script-per-boot.list_types() ...") + LOG.debug("in shell_script-per-boot.list_types() ...") return(["text/x-shellscript-per-boot"]) def handle_part(data, ctype, script_path, payload): if script_path is not None: - LOG.debug("### in shell_script-per-boot.handle_part() ...") - LOG.debug(f"### x-shellscript-per-boot.handle_part: script_path={script_path}") + LOG.debug("in shell_script-per-boot.handle_part() ...") + LOG.debug(f"x-shellscript-per-boot.handle_part: {script_path=}") (folder, filename) = os.path.split(script_path) - LOG.debug(f"### folder={folder} filename={filename}") + LOG.debug(f"{folder=} {filename=}") path = f"/var/lib/cloud/scripts/per-boot/{filename}" - LOG.debug(f"### path={path}") + LOG.debug(f"{path=}") util.write_file(path, payload, 0o700) diff --git a/cloudinit/cmd/devel/shell_script_per_instance.py b/cloudinit/cmd/devel/shell_script_per_instance.py index 257bbef4d35..75598cd92a4 100644 --- a/cloudinit/cmd/devel/shell_script_per_instance.py +++ b/cloudinit/cmd/devel/shell_script_per_instance.py @@ -11,16 +11,16 @@ def list_types(): # return a list of mime-types that are handled by this module - LOG.debug("### in shell_script-per-instance.list_types() ...") + LOG.debug("in shell_script-per-instance.list_types() ...") return(["text/x-shellscript-per-instance"]) def handle_part(data, ctype, script_path, payload): if script_path is not None: - LOG.debug("### in shell_script-per-instance.handle_part() ...") - LOG.debug(f"### x-shellscript-per-instance.handle_part: script_path={script_path}") + LOG.debug("in shell_script-per-instance.handle_part() ...") + LOG.debug(f"x-shellscript-per-instance.handle_part: {script_path=}") (folder, filename) = os.path.split(script_path) - LOG.debug(f"### folder={folder} filename={filename}") + LOG.debug(f"{folder=} {filename=}") path = f"/var/lib/cloud/scripts/per-instance/{filename}" - LOG.debug(f"### path={path}") + LOG.debug(f"{path=}") util.write_file(path, payload, 0o700) diff --git a/cloudinit/cmd/devel/shell_script_per_once.py b/cloudinit/cmd/devel/shell_script_per_once.py index 2a8d06427f8..f9d344f1505 100644 --- a/cloudinit/cmd/devel/shell_script_per_once.py +++ b/cloudinit/cmd/devel/shell_script_per_once.py @@ -11,16 +11,16 @@ def list_types(): # return a list of mime-types that are handled by this module - LOG.debug("### in shell_script-per-once.list_types() ...") + LOG.debug("in shell_script-per-once.list_types() ...") return(["text/x-shellscript-per-once"]) def handle_part(data, ctype, script_path, payload): if script_path is not None: - LOG.debug("### in shell_script-per-once.handle_part() ...") - LOG.debug(f"### x-shellscript-per-once.handle_part: script_path={script_path}") + LOG.debug("in shell_script-per-once.handle_part() ...") + LOG.debug(f"x-shellscript-per-once.handle_part: {script_path=}") (folder, filename) = os.path.split(script_path) - LOG.debug(f"### folder={folder} filename={filename}") + LOG.debug(f"{folder=} {filename=}) path = f"/var/lib/cloud/scripts/per-once/{filename}" - LOG.debug(f"### path={path}") + LOG.debug(f"{path=}") util.write_file(path, payload, 0o700) From ccb24357d34521f951cfe35089f658bc7613e4cc Mon Sep 17 00:00:00 2001 From: beantaxi Date: Fri, 1 Jan 2021 07:44:23 -0600 Subject: [PATCH 03/49] flake8 fixes part 2 (of 2?) --- cloudinit/cmd/devel/shell_script_per_boot.py | 7 +++---- cloudinit/cmd/devel/shell_script_per_instance.py | 7 +++---- cloudinit/cmd/devel/shell_script_per_once.py | 9 ++++----- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/cloudinit/cmd/devel/shell_script_per_boot.py b/cloudinit/cmd/devel/shell_script_per_boot.py index cf4efa42b59..5efc6a2b41f 100644 --- a/cloudinit/cmd/devel/shell_script_per_boot.py +++ b/cloudinit/cmd/devel/shell_script_per_boot.py @@ -1,16 +1,15 @@ -#part-handler +# part-handler # vi: syntax=python ts=4 +import os from cloudinit import log from cloudinit import util -import os -import pathlib LOG = log.getLogger(__name__) def list_types(): -# return a list of mime-types that are handled by this module + # return a list of mime-types that are handled by this module LOG.debug("in shell_script-per-boot.list_types() ...") return(["text/x-shellscript-per-boot"]) diff --git a/cloudinit/cmd/devel/shell_script_per_instance.py b/cloudinit/cmd/devel/shell_script_per_instance.py index 75598cd92a4..6e53a91b610 100644 --- a/cloudinit/cmd/devel/shell_script_per_instance.py +++ b/cloudinit/cmd/devel/shell_script_per_instance.py @@ -1,16 +1,15 @@ -#part-handler +# part-handler # vi: syntax=python ts=4 +import os from cloudinit import log from cloudinit import util -import os -import pathlib LOG = log.getLogger(__name__) def list_types(): -# return a list of mime-types that are handled by this module + # return a list of mime-types that are handled by this module LOG.debug("in shell_script-per-instance.list_types() ...") return(["text/x-shellscript-per-instance"]) diff --git a/cloudinit/cmd/devel/shell_script_per_once.py b/cloudinit/cmd/devel/shell_script_per_once.py index f9d344f1505..faccccc4cd5 100644 --- a/cloudinit/cmd/devel/shell_script_per_once.py +++ b/cloudinit/cmd/devel/shell_script_per_once.py @@ -1,16 +1,15 @@ -part-handler +# part-handler # vi: syntax=python ts=4 +import os from cloudinit import log from cloudinit import util -import os -import pathlib LOG = log.getLogger(__name__) def list_types(): -# return a list of mime-types that are handled by this module + # return a list of mime-types that are handled by this module LOG.debug("in shell_script-per-once.list_types() ...") return(["text/x-shellscript-per-once"]) @@ -20,7 +19,7 @@ def handle_part(data, ctype, script_path, payload): LOG.debug("in shell_script-per-once.handle_part() ...") LOG.debug(f"x-shellscript-per-once.handle_part: {script_path=}") (folder, filename) = os.path.split(script_path) - LOG.debug(f"{folder=} {filename=}) + LOG.debug(f"{folder=} {filename=}") path = f"/var/lib/cloud/scripts/per-once/{filename}" LOG.debug(f"{path=}") util.write_file(path, payload, 0o700) From 0e4d9de29c4d14695bd1bda4aeb56a098173d416 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Fri, 1 Jan 2021 08:07:36 -0600 Subject: [PATCH 04/49] pylint fixes --- cloudinit/cmd/devel/shell_script_per_boot.py | 8 ++++---- cloudinit/cmd/devel/shell_script_per_instance.py | 8 ++++---- cloudinit/cmd/devel/shell_script_per_once.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cloudinit/cmd/devel/shell_script_per_boot.py b/cloudinit/cmd/devel/shell_script_per_boot.py index 5efc6a2b41f..aada5e3c992 100644 --- a/cloudinit/cmd/devel/shell_script_per_boot.py +++ b/cloudinit/cmd/devel/shell_script_per_boot.py @@ -16,10 +16,10 @@ def list_types(): def handle_part(data, ctype, script_path, payload): if script_path is not None: - LOG.debug("in shell_script-per-boot.handle_part() ...") - LOG.debug(f"x-shellscript-per-boot.handle_part: {script_path=}") + LOG.info("in shell_script-per-boot.handle_part() ...") + LOG.debug("script_path=%s", script_path) (folder, filename) = os.path.split(script_path) - LOG.debug(f"{folder=} {filename=}") + LOG.debug("folder=%s filename=%s", folder, filename) path = f"/var/lib/cloud/scripts/per-boot/{filename}" - LOG.debug(f"{path=}") + LOG.debug("path=%s", path) util.write_file(path, payload, 0o700) diff --git a/cloudinit/cmd/devel/shell_script_per_instance.py b/cloudinit/cmd/devel/shell_script_per_instance.py index 6e53a91b610..8bebf7005f2 100644 --- a/cloudinit/cmd/devel/shell_script_per_instance.py +++ b/cloudinit/cmd/devel/shell_script_per_instance.py @@ -16,10 +16,10 @@ def list_types(): def handle_part(data, ctype, script_path, payload): if script_path is not None: - LOG.debug("in shell_script-per-instance.handle_part() ...") - LOG.debug(f"x-shellscript-per-instance.handle_part: {script_path=}") + LOG.info("in shell_script-per-instance.handle_part() ...") + LOG.debug("script_path=%s", script_path) (folder, filename) = os.path.split(script_path) - LOG.debug(f"{folder=} {filename=}") + LOG.debug("folder=%s filename=%s", folder, filename) path = f"/var/lib/cloud/scripts/per-instance/{filename}" - LOG.debug(f"{path=}") + LOG.debug("path=%s", path) util.write_file(path, payload, 0o700) diff --git a/cloudinit/cmd/devel/shell_script_per_once.py b/cloudinit/cmd/devel/shell_script_per_once.py index faccccc4cd5..aef10960fc1 100644 --- a/cloudinit/cmd/devel/shell_script_per_once.py +++ b/cloudinit/cmd/devel/shell_script_per_once.py @@ -16,10 +16,10 @@ def list_types(): def handle_part(data, ctype, script_path, payload): if script_path is not None: - LOG.debug("in shell_script-per-once.handle_part() ...") - LOG.debug(f"x-shellscript-per-once.handle_part: {script_path=}") + LOG.info("in shell_script-per-once.handle_part() ...") + LOG.debug("script_path=%s", script_path) (folder, filename) = os.path.split(script_path) - LOG.debug(f"{folder=} {filename=}") + LOG.debug("folder=%s filename=%s", folder, filename) path = f"/var/lib/cloud/scripts/per-once/{filename}" - LOG.debug(f"{path=}") + LOG.debug("path=%s", path) util.write_file(path, payload, 0o700) From 30fd096dca8e7fe1d10ba32ecf486da931e675e6 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Sat, 27 Feb 2021 23:13:35 -0600 Subject: [PATCH 05/49] ok lets do this --- .../handlers/shell_script_by_frequency.py | 58 +++++++++++++++++++ cloudinit/stages.py | 9 +++ 2 files changed, 67 insertions(+) create mode 100644 cloudinit/handlers/shell_script_by_frequency.py diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py new file mode 100644 index 00000000000..452cc435fa0 --- /dev/null +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -0,0 +1,58 @@ +# part-handler +# vi: syntax=python ts=4 + +import os +from cloudinit import log +from cloudinit import util + +LOG = log.getLogger(__name__) + + +def get_script_path_by_frequency () + +class ShellScriptPartHandler(handlers.Handler): + prefixes = ['#!'] + + def __init__(self, paths, **_kwargs): + handlers.Handler.__init__(self, PER_ALWAYS) + self.script_dir = paths.get_ipath_cur('scripts') + if 'script_path' in _kwargs: + self.script_dir = paths.get_ipath_cur(_kwargs['script_path']) + + def handle_part(self, data, ctype, filename, payload, frequency): + if ctype in handlers.CONTENT_SIGNALS: + # TODO(harlowja): maybe delete existing things here + return + + filename = util.clean_filename(filename) + payload = util.dos2unix(payload) + path = os.path.join(self.script_dir, filename) + util.write_file(path, payload, 0o700) + + +### per-boot +class ShellScriptPerBootPartHandler(handlers.Handler): + + def __init__(self, paths, **_kwargs): + handlers.Handler.__init__(self, PER_BOOstT) + self.script_dir = paths.get_ipath_cur('scripts') + if 'script_path' in _kwargs: + self.script_dir = paths.get_ipath_cur(_kwargs['script_path']) + + def list_types(): + return(["text/x-shellscript-per-boot"]) + + def handle_part(self, data, ctype, script_path, payload): + if script_path is not None: + LOG.debug("script_path=%s", script_path) + (folder, filename) = os.path.split(script_path) + LOG.debug("folder=%s filename=%s", folder, filename) + path = f"/var/lib/cloud/scripts/per-boot/{filename}" + LOG.debug("path=%s", path) + util.write_file(path, payload, 0o700) + +### per-instance + + +### per-once + diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 0cce6e8061c..b534530131f 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -19,6 +19,9 @@ from cloudinit.handlers.cloud_config import CloudConfigPartHandler from cloudinit.handlers.jinja_template import JinjaTemplatePartHandler from cloudinit.handlers.shell_script import ShellScriptPartHandler +from cloudinit.handlers.shell_script_by_frequency import ShellScriptPerBootPartHandler +from cloudinit.handlers.shell_script_by_frequency import ShellScriptPerInstancePartHandler +from cloudinit.handlers.shell_script_by_frequency import ShellScriptPerOncePartHandler from cloudinit.handlers.upstart_job import UpstartJobPartHandler from cloudinit.event import EventType @@ -415,9 +418,15 @@ def _default_handlers(self, opts=None): # TODO(harlowja) Hmmm, should we dynamically import these?? cloudconfig_handler = CloudConfigPartHandler(**opts) shellscript_handler = ShellScriptPartHandler(**opts) + shellscript_per_boot_handler = ShellScriptPerBootPartHandler(**opts) + shellscript_per_instance_handler = ShellScriptPerInstancePartHandler(**opts) + shellscript_per_once_handler = ShellScriptPerOncePartHandler(**opts) def_handlers = [ cloudconfig_handler, shellscript_handler, + shellscript_per_boot_handler, + shellscript_per_instance_handler, + shellscript_per_once_handler, BootHookPartHandler(**opts), UpstartJobPartHandler(**opts), ] From 0d9acbeda24c0def8f34255ad10cc2061f788189 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Sun, 28 Feb 2021 01:29:57 -0600 Subject: [PATCH 06/49] First commit of the changes suggested/requested by smoser and raharper --- .../handlers/shell_script_by_frequency.py | 80 +++++++++++-------- tests/unittests/test_builtin_handlers.py | 23 ++++++ 2 files changed, 69 insertions(+), 34 deletions(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index 452cc435fa0..d756bef4b91 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -4,55 +4,67 @@ import os from cloudinit import log from cloudinit import util - +from cloudinit import handlers +from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE +from cloudinit.cmd.devel import read_cfg_paths LOG = log.getLogger(__name__) - -def get_script_path_by_frequency () - -class ShellScriptPartHandler(handlers.Handler): - prefixes = ['#!'] - - def __init__(self, paths, **_kwargs): - handlers.Handler.__init__(self, PER_ALWAYS) - self.script_dir = paths.get_ipath_cur('scripts') - if 'script_path' in _kwargs: - self.script_dir = paths.get_ipath_cur(_kwargs['script_path']) - - def handle_part(self, data, ctype, filename, payload, frequency): - if ctype in handlers.CONTENT_SIGNALS: - # TODO(harlowja): maybe delete existing things here - return - - filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - path = os.path.join(self.script_dir, filename) - util.write_file(path, payload, 0o700) - +pathMap = { + PER_ALWAYS: 'per-boot', + PER_INSTANCE: 'per-instance', + PER_ONCE: 'per-once' +} + +def get_script_folder_by_frequency (freq): + freqPath = pathMap[freq] + ci_paths = read_cfg_paths() + scripts_dir = ci_paths.get_cpath('scripts') # defaults to /var/lib/cloud/ + scripts + folder = os.path.join(scripts_dir, freqPath) + return folder + +def write_script_by_frequency (script_path, payload, frequency): + filename = os.path.basename(script_path) + filename = util.clean_filename(filename) + folder = get_script_folder_by_frequency(frequency) + path = os.path.join(folder, filename) + payload = util.dos2unix(payload) + util.write_file(path, payload, 0o700) ### per-boot class ShellScriptPerBootPartHandler(handlers.Handler): - def __init__(self, paths, **_kwargs): - handlers.Handler.__init__(self, PER_BOOstT) - self.script_dir = paths.get_ipath_cur('scripts') - if 'script_path' in _kwargs: - self.script_dir = paths.get_ipath_cur(_kwargs['script_path']) + handlers.Handler.__init__(self, PER_ALWAYS) - def list_types(): + def list_types(self): return(["text/x-shellscript-per-boot"]) def handle_part(self, data, ctype, script_path, payload): if script_path is not None: LOG.debug("script_path=%s", script_path) - (folder, filename) = os.path.split(script_path) - LOG.debug("folder=%s filename=%s", folder, filename) - path = f"/var/lib/cloud/scripts/per-boot/{filename}" - LOG.debug("path=%s", path) - util.write_file(path, payload, 0o700) + write_script_by_frequency(script_path, payload, PER_ALWAYS) ### per-instance +class ShellScriptPerInstancePartHandler(handlers.Handler): + def __init__(self, paths, **_kwargs): + handlers.Handler.__init__(self, PER_INSTANCE) + def list_types(self): + return(["text/x-shellscript-per-boot"]) + + def handle_part(self, data, ctype, script_path, payload): + if script_path is not None: + LOG.debug("script_path=%s", script_path) + write_script_by_frequency(script_path, payload, PER_INSTANCE) ### per-once +class ShellScriptPerOPncePartHandler(handlers.Handler): + def __init__(self, paths, **_kwargs): + handlers.Handler.__init__(self, PER_ONCE) + def list_types(self): + return(["text/x-shellscript-per-boot"]) + + def handle_part(self, data, ctype, script_path, payload): + if script_path is not None: + LOG.debug("script_path=%s", script_path) + write_script_by_frequency(script_path, payload, PER_ONCE) \ No newline at end of file diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index c5675249cf7..043e36eefb4 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -380,4 +380,27 @@ def test_render_jinja_payload_replaces_missing_variables_and_warns(self): " 'myfile': 'NOTHERE'") self.assertIn(expected_log, self.logs.getvalue()) + +class TestShellScriptByFrequencyHandlers(CiTestCase): + with_logs = True + + def test_frequency(self, frequency) + from cloudinit.cmd.devel import read_cfg_paths + from handlers import shell_script_by_frequency + ci_paths = read_cfg_paths() + scripts_dir = ci_paths.get_cpath('scripts') # defaults to /var/lib/cloud/ + scripts + testFolder = os.path.join(scripts_dir, shell_script_by_frequency.pathMap[frequency]) + folder = shell_script_by_frequency.get_script_folder_by_frequency(frequency) + self.assertEqual(testFolder, folder) + + def test_get_script_folder_per_boot(self): + self.test_frequency('per-boot') + + def test_get_script_folder_per_instance(self): + self.test_frequency('per-instance') + + def test_get_script_folder_per_once(self): + self.test_frequency('per-once') + + # vi: ts=4 expandtab From d511f7d0233a211d356420bf75e753851ad09ba5 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Sun, 28 Feb 2021 02:12:56 -0600 Subject: [PATCH 07/49] Change scripts_dir initialization to match shell_script.py to fix circular dependency --- .../handlers/shell_script_by_frequency.py | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index d756bef4b91..84046146ce0 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -4,9 +4,8 @@ import os from cloudinit import log from cloudinit import util -from cloudinit import handlers +from cloudinit.handlers import Handler from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE -from cloudinit.cmd.devel import read_cfg_paths LOG = log.getLogger(__name__) pathMap = { @@ -15,25 +14,26 @@ PER_ONCE: 'per-once' } -def get_script_folder_by_frequency (freq): +def get_script_folder_by_frequency (scripts_dir, freq): freqPath = pathMap[freq] - ci_paths = read_cfg_paths() - scripts_dir = ci_paths.get_cpath('scripts') # defaults to /var/lib/cloud/ + scripts folder = os.path.join(scripts_dir, freqPath) return folder -def write_script_by_frequency (script_path, payload, frequency): +def write_script_by_frequency (script_path, payload, frequency, scripts_dir): filename = os.path.basename(script_path) filename = util.clean_filename(filename) - folder = get_script_folder_by_frequency(frequency) + folder = get_script_folder_by_frequency(scripts_dir, frequency) path = os.path.join(folder, filename) payload = util.dos2unix(payload) util.write_file(path, payload, 0o700) ### per-boot -class ShellScriptPerBootPartHandler(handlers.Handler): +class ShellScriptPerBootPartHandler(Handler): def __init__(self, paths, **_kwargs): - handlers.Handler.__init__(self, PER_ALWAYS) + Handler.__init__(self, PER_ALWAYS) + self.scripts_dir = paths.get_ipath_cur('scripts') + if 'script_path' in _kwargs: + self.scripts_dir = paths.get_ipath_cur(_kwargs['script_path']) def list_types(self): return(["text/x-shellscript-per-boot"]) @@ -41,12 +41,15 @@ def list_types(self): def handle_part(self, data, ctype, script_path, payload): if script_path is not None: LOG.debug("script_path=%s", script_path) - write_script_by_frequency(script_path, payload, PER_ALWAYS) + write_script_by_frequency(script_path, payload, PER_ALWAYS, self.scripts_dir) ### per-instance -class ShellScriptPerInstancePartHandler(handlers.Handler): +class ShellScriptPerInstancePartHandler(Handler): def __init__(self, paths, **_kwargs): - handlers.Handler.__init__(self, PER_INSTANCE) + Handler.__init__(self, PER_INSTANCE) + self.scripts_dir = paths.get_ipath_cur('scripts') + if 'script_path' in _kwargs: + self.scripts_dir = paths.get_ipath_cur(_kwargs['script_path']) def list_types(self): return(["text/x-shellscript-per-boot"]) @@ -54,12 +57,15 @@ def list_types(self): def handle_part(self, data, ctype, script_path, payload): if script_path is not None: LOG.debug("script_path=%s", script_path) - write_script_by_frequency(script_path, payload, PER_INSTANCE) + write_script_by_frequency(script_path, payload, PER_INSTANCE, self.scripts_dir) ### per-once -class ShellScriptPerOPncePartHandler(handlers.Handler): +class ShellScriptPerOPncePartHandler(Handler): def __init__(self, paths, **_kwargs): - handlers.Handler.__init__(self, PER_ONCE) + Handler.__init__(self, PER_ONCE) + self.scripts_dir = paths.get_ipath_cur('scripts') + if 'script_path' in _kwargs: + self.scripts_dir = paths.get_ipath_cur(_kwargs['script_path']) def list_types(self): return(["text/x-shellscript-per-boot"]) @@ -67,4 +73,4 @@ def list_types(self): def handle_part(self, data, ctype, script_path, payload): if script_path is not None: LOG.debug("script_path=%s", script_path) - write_script_by_frequency(script_path, payload, PER_ONCE) \ No newline at end of file + write_script_by_frequency(script_path, payload, PER_ONCE, self.scripts_dir) \ No newline at end of file From 56ea2cc2aeda933921c0378fadb4bb97f19946f9 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Sun, 28 Feb 2021 02:32:02 -0600 Subject: [PATCH 08/49] fixed typos --- cloudinit/handlers/shell_script_by_frequency.py | 2 +- tests/unittests/test_builtin_handlers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index 84046146ce0..111cfb85c8b 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -60,7 +60,7 @@ def handle_part(self, data, ctype, script_path, payload): write_script_by_frequency(script_path, payload, PER_INSTANCE, self.scripts_dir) ### per-once -class ShellScriptPerOPncePartHandler(Handler): +class ShellScriptPerOncePartHandler(Handler): def __init__(self, paths, **_kwargs): Handler.__init__(self, PER_ONCE) self.scripts_dir = paths.get_ipath_cur('scripts') diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 043e36eefb4..540bb2dbc76 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -384,7 +384,7 @@ def test_render_jinja_payload_replaces_missing_variables_and_warns(self): class TestShellScriptByFrequencyHandlers(CiTestCase): with_logs = True - def test_frequency(self, frequency) + def test_frequency(self, frequency): from cloudinit.cmd.devel import read_cfg_paths from handlers import shell_script_by_frequency ci_paths = read_cfg_paths() From 45e13204b96eb01fda5d23e3805760e9964fcad8 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Sun, 28 Feb 2021 02:39:36 -0600 Subject: [PATCH 09/49] more fixes --- tests/unittests/test_builtin_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 540bb2dbc76..964ebd75703 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -386,11 +386,11 @@ class TestShellScriptByFrequencyHandlers(CiTestCase): def test_frequency(self, frequency): from cloudinit.cmd.devel import read_cfg_paths - from handlers import shell_script_by_frequency + from cloudinit.handlers import shell_script_by_frequency ci_paths = read_cfg_paths() scripts_dir = ci_paths.get_cpath('scripts') # defaults to /var/lib/cloud/ + scripts testFolder = os.path.join(scripts_dir, shell_script_by_frequency.pathMap[frequency]) - folder = shell_script_by_frequency.get_script_folder_by_frequency(frequency) + folder = shell_script_by_frequency.get_script_folder_by_frequency(frequency, scripts_dir) self.assertEqual(testFolder, folder) def test_get_script_folder_per_boot(self): From 3439b256e1672c4f720aa2c3fd0176aff1fb1dd9 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Sun, 28 Feb 2021 03:11:12 -0600 Subject: [PATCH 10/49] flake8 fixes --- .../handlers/shell_script_by_frequency.py | 24 ++++++++++++------- cloudinit/stages.py | 10 ++++---- tests/unittests/test_builtin_handlers.py | 18 +++++++------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index 111cfb85c8b..3ae78c18b83 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -14,12 +14,14 @@ PER_ONCE: 'per-once' } -def get_script_folder_by_frequency (scripts_dir, freq): + +def get_script_folder_by_frequency(scripts_dir, freq): freqPath = pathMap[freq] folder = os.path.join(scripts_dir, freqPath) return folder -def write_script_by_frequency (script_path, payload, frequency, scripts_dir): + +def write_script_by_frequency(script_path, payload, frequency, scripts_dir): filename = os.path.basename(script_path) filename = util.clean_filename(filename) folder = get_script_folder_by_frequency(scripts_dir, frequency) @@ -27,7 +29,8 @@ def write_script_by_frequency (script_path, payload, frequency, scripts_dir): payload = util.dos2unix(payload) util.write_file(path, payload, 0o700) -### per-boot + +# per-boot class ShellScriptPerBootPartHandler(Handler): def __init__(self, paths, **_kwargs): Handler.__init__(self, PER_ALWAYS) @@ -41,9 +44,11 @@ def list_types(self): def handle_part(self, data, ctype, script_path, payload): if script_path is not None: LOG.debug("script_path=%s", script_path) - write_script_by_frequency(script_path, payload, PER_ALWAYS, self.scripts_dir) + write_script_by_frequency(script_path, payload, PER_ALWAYS, + self.scripts_dir) -### per-instance + +# per-instance class ShellScriptPerInstancePartHandler(Handler): def __init__(self, paths, **_kwargs): Handler.__init__(self, PER_INSTANCE) @@ -57,9 +62,11 @@ def list_types(self): def handle_part(self, data, ctype, script_path, payload): if script_path is not None: LOG.debug("script_path=%s", script_path) - write_script_by_frequency(script_path, payload, PER_INSTANCE, self.scripts_dir) + write_script_by_frequency(script_path, payload, PER_INSTANCE, + self.scripts_dir) + -### per-once +# per-once class ShellScriptPerOncePartHandler(Handler): def __init__(self, paths, **_kwargs): Handler.__init__(self, PER_ONCE) @@ -73,4 +80,5 @@ def list_types(self): def handle_part(self, data, ctype, script_path, payload): if script_path is not None: LOG.debug("script_path=%s", script_path) - write_script_by_frequency(script_path, payload, PER_ONCE, self.scripts_dir) \ No newline at end of file + write_script_by_frequency(script_path, payload, PER_ONCE, + self.scripts_dir) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 8015db4723f..1cf38081104 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -19,9 +19,10 @@ from cloudinit.handlers.cloud_config import CloudConfigPartHandler from cloudinit.handlers.jinja_template import JinjaTemplatePartHandler from cloudinit.handlers.shell_script import ShellScriptPartHandler -from cloudinit.handlers.shell_script_by_frequency import ShellScriptPerBootPartHandler -from cloudinit.handlers.shell_script_by_frequency import ShellScriptPerInstancePartHandler -from cloudinit.handlers.shell_script_by_frequency import ShellScriptPerOncePartHandler +from cloudinit.handlers.shell_script_by_frequency import \ + (ShellScriptPerBootPartHandler, + ShellScriptPerInstancePartHandler, + ShellScriptPerOncePartHandler) from cloudinit.handlers.upstart_job import UpstartJobPartHandler from cloudinit.event import EventType @@ -419,7 +420,8 @@ def _default_handlers(self, opts=None): cloudconfig_handler = CloudConfigPartHandler(**opts) shellscript_handler = ShellScriptPartHandler(**opts) shellscript_per_boot_handler = ShellScriptPerBootPartHandler(**opts) - shellscript_per_instance_handler = ShellScriptPerInstancePartHandler(**opts) + shellscript_per_instance_handler = \ + ShellScriptPerInstancePartHandler(**opts) shellscript_per_once_handler = ShellScriptPerOncePartHandler(**opts) def_handlers = [ cloudconfig_handler, diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 964ebd75703..0e80fd05231 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -384,23 +384,25 @@ def test_render_jinja_payload_replaces_missing_variables_and_warns(self): class TestShellScriptByFrequencyHandlers(CiTestCase): with_logs = True - def test_frequency(self, frequency): + def do_test_frequency(self, frequency): from cloudinit.cmd.devel import read_cfg_paths - from cloudinit.handlers import shell_script_by_frequency + from cloudinit.handlers.shell_script_by_frequency \ + import (get_script_folder_by_frequency, + pathMap) ci_paths = read_cfg_paths() - scripts_dir = ci_paths.get_cpath('scripts') # defaults to /var/lib/cloud/ + scripts - testFolder = os.path.join(scripts_dir, shell_script_by_frequency.pathMap[frequency]) - folder = shell_script_by_frequency.get_script_folder_by_frequency(frequency, scripts_dir) + scripts_dir = ci_paths.get_cpath('scripts') + testFolder = os.path.join(scripts_dir, pathMap[frequency]) + folder = get_script_folder_by_frequency(frequency, scripts_dir) self.assertEqual(testFolder, folder) def test_get_script_folder_per_boot(self): - self.test_frequency('per-boot') + self.do_test_frequency('per-boot') def test_get_script_folder_per_instance(self): - self.test_frequency('per-instance') + self.do_test_frequency('per-instance') def test_get_script_folder_per_once(self): - self.test_frequency('per-once') + self.do_test_frequency('per-once') # vi: ts=4 expandtab From 888e70b754a58dd605251287a4c03ea390a97d51 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Sun, 28 Feb 2021 03:26:58 -0600 Subject: [PATCH 11/49] pytest fixes --- cloudinit/handlers/shell_script_by_frequency.py | 4 ++-- tests/unittests/test_builtin_handlers.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index 3ae78c18b83..0dfbe6b2bb6 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -15,7 +15,7 @@ } -def get_script_folder_by_frequency(scripts_dir, freq): +def get_script_folder_by_frequency(freq, scripts_dir): freqPath = pathMap[freq] folder = os.path.join(scripts_dir, freqPath) return folder @@ -24,7 +24,7 @@ def get_script_folder_by_frequency(scripts_dir, freq): def write_script_by_frequency(script_path, payload, frequency, scripts_dir): filename = os.path.basename(script_path) filename = util.clean_filename(filename) - folder = get_script_folder_by_frequency(scripts_dir, frequency) + folder = get_script_folder_by_frequency(frequency, scripts_dir) path = os.path.join(folder, filename) payload = util.dos2unix(payload) util.write_file(path, payload, 0o700) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 0e80fd05231..00148b6bf72 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -25,7 +25,7 @@ from cloudinit.handlers.shell_script import ShellScriptPartHandler from cloudinit.handlers.upstart_job import UpstartJobPartHandler -from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, PER_ONCE) class TestUpstartJobPartHandler(FilesystemMockingTestCase): @@ -396,13 +396,13 @@ def do_test_frequency(self, frequency): self.assertEqual(testFolder, folder) def test_get_script_folder_per_boot(self): - self.do_test_frequency('per-boot') + self.do_test_frequency(PER_ALWAYS) def test_get_script_folder_per_instance(self): - self.do_test_frequency('per-instance') + self.do_test_frequency(PER_INSTANCE) def test_get_script_folder_per_once(self): - self.do_test_frequency('per-once') + self.do_test_frequency(PER_ONCE) # vi: ts=4 expandtab From a834b2e43d5d2952da9f6b96756655837b78c825 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Sun, 28 Feb 2021 08:54:57 -0600 Subject: [PATCH 12/49] pytest fixes --- cloudinit/handlers/shell_script_by_frequency.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index 0dfbe6b2bb6..a901ad620bc 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -41,7 +41,7 @@ def __init__(self, paths, **_kwargs): def list_types(self): return(["text/x-shellscript-per-boot"]) - def handle_part(self, data, ctype, script_path, payload): + def handle_part(self, data, ctype, script_path, payload, frequency): if script_path is not None: LOG.debug("script_path=%s", script_path) write_script_by_frequency(script_path, payload, PER_ALWAYS, @@ -59,7 +59,7 @@ def __init__(self, paths, **_kwargs): def list_types(self): return(["text/x-shellscript-per-boot"]) - def handle_part(self, data, ctype, script_path, payload): + def handle_part(self, data, ctype, script_path, payload, frequency): if script_path is not None: LOG.debug("script_path=%s", script_path) write_script_by_frequency(script_path, payload, PER_INSTANCE, @@ -77,7 +77,7 @@ def __init__(self, paths, **_kwargs): def list_types(self): return(["text/x-shellscript-per-boot"]) - def handle_part(self, data, ctype, script_path, payload): + def handle_part(self, data, ctype, script_path, payload, frequency): if script_path is not None: LOG.debug("script_path=%s", script_path) write_script_by_frequency(script_path, payload, PER_ONCE, From 87fddb9eea738dc758be6b0474b94d9ab60d557c Mon Sep 17 00:00:00 2001 From: beantaxi Date: Thu, 4 Mar 2021 09:55:53 -0600 Subject: [PATCH 13/49] Changed to get_cpath() and added some comments --- .../handlers/shell_script_by_frequency.py | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index a901ad620bc..a09d54b5b2d 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -8,6 +8,11 @@ from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE LOG = log.getLogger(__name__) +# cloutinit/settings.py defines PER_*** frequency constants. It makes sense to +# use them here, instead hardcodes, and map them to the 'per-***' frequency- +# specific folders in /v/l/c/scripts. It might make sense to expose this at a +# higher level or in a more general module -- eg maybe in cloudinit/settings.py +# itself -- but for now it's here. pathMap = { PER_ALWAYS: 'per-boot', PER_INSTANCE: 'per-instance', @@ -15,12 +20,16 @@ } +# Using pathMap (defined above), return the frequency-specific subfolder for a +# given frequency constant and parent folder. def get_script_folder_by_frequency(freq, scripts_dir): freqPath = pathMap[freq] folder = os.path.join(scripts_dir, freqPath) return folder +# Given a filename, a payload, a frequency, and a scripts folder, write the +# payload to the correct frequency-specific paths def write_script_by_frequency(script_path, payload, frequency, scripts_dir): filename = os.path.basename(script_path) filename = util.clean_filename(filename) @@ -34,9 +43,9 @@ def write_script_by_frequency(script_path, payload, frequency, scripts_dir): class ShellScriptPerBootPartHandler(Handler): def __init__(self, paths, **_kwargs): Handler.__init__(self, PER_ALWAYS) - self.scripts_dir = paths.get_ipath_cur('scripts') + self.scripts_dir = paths.get_cpath('scripts') if 'script_path' in _kwargs: - self.scripts_dir = paths.get_ipath_cur(_kwargs['script_path']) + self.scripts_dir = paths.get_cpath(_kwargs['script_path']) def list_types(self): return(["text/x-shellscript-per-boot"]) @@ -44,6 +53,8 @@ def list_types(self): def handle_part(self, data, ctype, script_path, payload, frequency): if script_path is not None: LOG.debug("script_path=%s", script_path) + filename = os.path.basename(script_path) + filename = util.clean_filename(filename) write_script_by_frequency(script_path, payload, PER_ALWAYS, self.scripts_dir) @@ -52,9 +63,9 @@ def handle_part(self, data, ctype, script_path, payload, frequency): class ShellScriptPerInstancePartHandler(Handler): def __init__(self, paths, **_kwargs): Handler.__init__(self, PER_INSTANCE) - self.scripts_dir = paths.get_ipath_cur('scripts') + self.scripts_dir = paths.get_cpath('scripts') if 'script_path' in _kwargs: - self.scripts_dir = paths.get_ipath_cur(_kwargs['script_path']) + self.scripts_dir = paths.get_cpath(_kwargs['script_path']) def list_types(self): return(["text/x-shellscript-per-boot"]) @@ -62,7 +73,9 @@ def list_types(self): def handle_part(self, data, ctype, script_path, payload, frequency): if script_path is not None: LOG.debug("script_path=%s", script_path) - write_script_by_frequency(script_path, payload, PER_INSTANCE, + filename = os.path.basename(script_path) + filename = util.clean_filename(filename) + write_script_by_frequency(filename, payload, PER_INSTANCE, self.scripts_dir) @@ -70,9 +83,9 @@ def handle_part(self, data, ctype, script_path, payload, frequency): class ShellScriptPerOncePartHandler(Handler): def __init__(self, paths, **_kwargs): Handler.__init__(self, PER_ONCE) - self.scripts_dir = paths.get_ipath_cur('scripts') + self.scripts_dir = paths.get_cpath('scripts') if 'script_path' in _kwargs: - self.scripts_dir = paths.get_ipath_cur(_kwargs['script_path']) + self.scripts_dir = paths.get_cpath(_kwargs['script_path']) def list_types(self): return(["text/x-shellscript-per-boot"]) @@ -80,5 +93,7 @@ def list_types(self): def handle_part(self, data, ctype, script_path, payload, frequency): if script_path is not None: LOG.debug("script_path=%s", script_path) + filename = os.path.basename(script_path) + filename = util.clean_filename(filename) write_script_by_frequency(script_path, payload, PER_ONCE, self.scripts_dir) From f33429eb7916cbcb9f6a771daf608a1d2a30e090 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Thu, 4 Mar 2021 10:47:44 -0600 Subject: [PATCH 14/49] Common base class for the part handlers. Required # pylint:disable; might need an adult. --- .../handlers/shell_script_by_frequency.py | 59 ++++++++----------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index a09d54b5b2d..8b72dcd0ae9 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -39,17 +39,16 @@ def write_script_by_frequency(script_path, payload, frequency, scripts_dir): util.write_file(path, payload, 0o700) -# per-boot -class ShellScriptPerBootPartHandler(Handler): - def __init__(self, paths, **_kwargs): - Handler.__init__(self, PER_ALWAYS) +# This is purely to allow packaging args up into a single object to please +# pylint and avoid 'too many positional args' complaints. I'd be happy to +# have an alernative. +class ShellScriptByFreqPartHandler(Handler): + def __init__(self, paths, freq, **_kwargs): + Handler.__init__(self, freq) self.scripts_dir = paths.get_cpath('scripts') if 'script_path' in _kwargs: self.scripts_dir = paths.get_cpath(_kwargs['script_path']) - def list_types(self): - return(["text/x-shellscript-per-boot"]) - def handle_part(self, data, ctype, script_path, payload, frequency): if script_path is not None: LOG.debug("script_path=%s", script_path) @@ -59,41 +58,35 @@ def handle_part(self, data, ctype, script_path, payload, frequency): self.scripts_dir) -# per-instance -class ShellScriptPerInstancePartHandler(Handler): +# per-boot +class ShellScriptPerBootPartHandler(ShellScriptByFreqPartHandler): def __init__(self, paths, **_kwargs): - Handler.__init__(self, PER_INSTANCE) - self.scripts_dir = paths.get_cpath('scripts') - if 'script_path' in _kwargs: - self.scripts_dir = paths.get_cpath(_kwargs['script_path']) + # pylint: disable=too-many-function-args + ShellScriptByFreqPartHandler.__init__(self, paths, PER_ALWAYS, + **_kwargs) def list_types(self): return(["text/x-shellscript-per-boot"]) - def handle_part(self, data, ctype, script_path, payload, frequency): - if script_path is not None: - LOG.debug("script_path=%s", script_path) - filename = os.path.basename(script_path) - filename = util.clean_filename(filename) - write_script_by_frequency(filename, payload, PER_INSTANCE, - self.scripts_dir) + +# per-instance +class ShellScriptPerInstancePartHandler(ShellScriptByFreqPartHandler): + def __init__(self, paths, **_kwargs): + # pylint: disable=too-many-function-args + ShellScriptByFreqPartHandler.__init__(self, paths, PER_INSTANCE, + **_kwargs) + + def list_types(self): + # pylint: disable=too-many-function-args + return(["text/x-shellscript-per-boot"]) # per-once -class ShellScriptPerOncePartHandler(Handler): +class ShellScriptPerOncePartHandler(ShellScriptByFreqPartHandler): def __init__(self, paths, **_kwargs): - Handler.__init__(self, PER_ONCE) - self.scripts_dir = paths.get_cpath('scripts') - if 'script_path' in _kwargs: - self.scripts_dir = paths.get_cpath(_kwargs['script_path']) + # pylint: disable=too-many-function-args + ShellScriptByFreqPartHandler.__init__(self, paths, PER_ONCE, + **_kwargs) def list_types(self): return(["text/x-shellscript-per-boot"]) - - def handle_part(self, data, ctype, script_path, payload, frequency): - if script_path is not None: - LOG.debug("script_path=%s", script_path) - filename = os.path.basename(script_path) - filename = util.clean_filename(filename) - write_script_by_frequency(script_path, payload, PER_ONCE, - self.scripts_dir) From b7f651853cc9e6c7fd00a6e6e6b8e413f8606626 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Thu, 4 Mar 2021 11:10:08 -0600 Subject: [PATCH 15/49] using prefixes for list_types now --- cloudinit/handlers/__init__.py | 4 ++++ cloudinit/handlers/shell_script_by_frequency.py | 16 ++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index a409ff8a386..8d971cc21b0 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -51,6 +51,10 @@ '#cloud-config-archive': 'text/cloud-config-archive', '#cloud-config-jsonp': 'text/cloud-config-jsonp', '## template: jinja': 'text/jinja2', + 'text/x-shellscript-per-boot': 'text/x-shellscript-per-boot', + 'text/x-shellscript-per-instance': 'text/x-shellscript-per-instance', + 'text/x-shellscript-per-once': 'text/x-shellscript-per-once' + } # Sorted longest first diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index 8b72dcd0ae9..17a4a441f24 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -60,33 +60,29 @@ def handle_part(self, data, ctype, script_path, payload, frequency): # per-boot class ShellScriptPerBootPartHandler(ShellScriptByFreqPartHandler): + prefixes = ["text/x-shellscript-per-boot"] + def __init__(self, paths, **_kwargs): # pylint: disable=too-many-function-args ShellScriptByFreqPartHandler.__init__(self, paths, PER_ALWAYS, **_kwargs) - def list_types(self): - return(["text/x-shellscript-per-boot"]) - # per-instance class ShellScriptPerInstancePartHandler(ShellScriptByFreqPartHandler): + prefixes = ["text/x-shellscript-per-instance"] + def __init__(self, paths, **_kwargs): # pylint: disable=too-many-function-args ShellScriptByFreqPartHandler.__init__(self, paths, PER_INSTANCE, **_kwargs) - def list_types(self): - # pylint: disable=too-many-function-args - return(["text/x-shellscript-per-boot"]) - # per-once class ShellScriptPerOncePartHandler(ShellScriptByFreqPartHandler): + prefixes = ["text/x-shellscript-per-once"] + def __init__(self, paths, **_kwargs): # pylint: disable=too-many-function-args ShellScriptByFreqPartHandler.__init__(self, paths, PER_ONCE, **_kwargs) - - def list_types(self): - return(["text/x-shellscript-per-boot"]) From ab9d2381c9b5b766aaa67d8884728e9ed9b79b6c Mon Sep 17 00:00:00 2001 From: beantaxi Date: Fri, 5 Mar 2021 13:42:11 -0600 Subject: [PATCH 16/49] Changes suggested by TheRealFalcon --- cloudinit/cmd/devel/shell_script_per_boot.py | 25 ------------------- .../cmd/devel/shell_script_per_instance.py | 25 ------------------- cloudinit/cmd/devel/shell_script_per_once.py | 25 ------------------- cloudinit/handlers/__init__.py | 4 ++- .../handlers/shell_script_by_frequency.py | 20 ++++++--------- 5 files changed, 11 insertions(+), 88 deletions(-) delete mode 100644 cloudinit/cmd/devel/shell_script_per_boot.py delete mode 100644 cloudinit/cmd/devel/shell_script_per_instance.py delete mode 100644 cloudinit/cmd/devel/shell_script_per_once.py diff --git a/cloudinit/cmd/devel/shell_script_per_boot.py b/cloudinit/cmd/devel/shell_script_per_boot.py deleted file mode 100644 index aada5e3c992..00000000000 --- a/cloudinit/cmd/devel/shell_script_per_boot.py +++ /dev/null @@ -1,25 +0,0 @@ -# part-handler -# vi: syntax=python ts=4 - -import os -from cloudinit import log -from cloudinit import util - -LOG = log.getLogger(__name__) - - -def list_types(): - # return a list of mime-types that are handled by this module - LOG.debug("in shell_script-per-boot.list_types() ...") - return(["text/x-shellscript-per-boot"]) - - -def handle_part(data, ctype, script_path, payload): - if script_path is not None: - LOG.info("in shell_script-per-boot.handle_part() ...") - LOG.debug("script_path=%s", script_path) - (folder, filename) = os.path.split(script_path) - LOG.debug("folder=%s filename=%s", folder, filename) - path = f"/var/lib/cloud/scripts/per-boot/{filename}" - LOG.debug("path=%s", path) - util.write_file(path, payload, 0o700) diff --git a/cloudinit/cmd/devel/shell_script_per_instance.py b/cloudinit/cmd/devel/shell_script_per_instance.py deleted file mode 100644 index 8bebf7005f2..00000000000 --- a/cloudinit/cmd/devel/shell_script_per_instance.py +++ /dev/null @@ -1,25 +0,0 @@ -# part-handler -# vi: syntax=python ts=4 - -import os -from cloudinit import log -from cloudinit import util - -LOG = log.getLogger(__name__) - - -def list_types(): - # return a list of mime-types that are handled by this module - LOG.debug("in shell_script-per-instance.list_types() ...") - return(["text/x-shellscript-per-instance"]) - - -def handle_part(data, ctype, script_path, payload): - if script_path is not None: - LOG.info("in shell_script-per-instance.handle_part() ...") - LOG.debug("script_path=%s", script_path) - (folder, filename) = os.path.split(script_path) - LOG.debug("folder=%s filename=%s", folder, filename) - path = f"/var/lib/cloud/scripts/per-instance/{filename}" - LOG.debug("path=%s", path) - util.write_file(path, payload, 0o700) diff --git a/cloudinit/cmd/devel/shell_script_per_once.py b/cloudinit/cmd/devel/shell_script_per_once.py deleted file mode 100644 index aef10960fc1..00000000000 --- a/cloudinit/cmd/devel/shell_script_per_once.py +++ /dev/null @@ -1,25 +0,0 @@ -# part-handler -# vi: syntax=python ts=4 - -import os -from cloudinit import log -from cloudinit import util - -LOG = log.getLogger(__name__) - - -def list_types(): - # return a list of mime-types that are handled by this module - LOG.debug("in shell_script-per-once.list_types() ...") - return(["text/x-shellscript-per-once"]) - - -def handle_part(data, ctype, script_path, payload): - if script_path is not None: - LOG.info("in shell_script-per-once.handle_part() ...") - LOG.debug("script_path=%s", script_path) - (folder, filename) = os.path.split(script_path) - LOG.debug("folder=%s filename=%s", folder, filename) - path = f"/var/lib/cloud/scripts/per-once/{filename}" - LOG.debug("path=%s", path) - util.write_file(path, payload, 0o700) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 8d971cc21b0..45b6c7105c5 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -51,10 +51,12 @@ '#cloud-config-archive': 'text/cloud-config-archive', '#cloud-config-jsonp': 'text/cloud-config-jsonp', '## template: jinja': 'text/jinja2', + # Note: for the next 3 entries, the prefix doesn't matter because these are for + # types that can only be used as part of a MIME message. However, including these + # entries supresses warnings during `cloudinit devel make-mime` 'text/x-shellscript-per-boot': 'text/x-shellscript-per-boot', 'text/x-shellscript-per-instance': 'text/x-shellscript-per-instance', 'text/x-shellscript-per-once': 'text/x-shellscript-per-once' - } # Sorted longest first diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index 17a4a441f24..ed31853466e 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -1,6 +1,3 @@ -# part-handler -# vi: syntax=python ts=4 - import os from cloudinit import log from cloudinit import util @@ -20,17 +17,17 @@ } -# Using pathMap (defined above), return the frequency-specific subfolder for a -# given frequency constant and parent folder. def get_script_folder_by_frequency(freq, scripts_dir): + """Return the frequency-specific subfolder for a given frequency constant + and parent folder.""" freqPath = pathMap[freq] folder = os.path.join(scripts_dir, freqPath) return folder -# Given a filename, a payload, a frequency, and a scripts folder, write the -# payload to the correct frequency-specific paths def write_script_by_frequency(script_path, payload, frequency, scripts_dir): + """Given a filename, a payload, a frequency, and a scripts folder, write + the payload to the correct frequency-specific path""" filename = os.path.basename(script_path) filename = util.clean_filename(filename) folder = get_script_folder_by_frequency(frequency, scripts_dir) @@ -38,11 +35,8 @@ def write_script_by_frequency(script_path, payload, frequency, scripts_dir): payload = util.dos2unix(payload) util.write_file(path, payload, 0o700) - -# This is purely to allow packaging args up into a single object to please -# pylint and avoid 'too many positional args' complaints. I'd be happy to -# have an alernative. class ShellScriptByFreqPartHandler(Handler): + """Common base class for the frequency-specific script handlers.""" def __init__(self, paths, freq, **_kwargs): Handler.__init__(self, freq) self.scripts_dir = paths.get_cpath('scripts') @@ -54,7 +48,7 @@ def handle_part(self, data, ctype, script_path, payload, frequency): LOG.debug("script_path=%s", script_path) filename = os.path.basename(script_path) filename = util.clean_filename(filename) - write_script_by_frequency(script_path, payload, PER_ALWAYS, + write_script_by_frequency(script_path, payload, self.frequency, self.scripts_dir) @@ -86,3 +80,5 @@ def __init__(self, paths, **_kwargs): # pylint: disable=too-many-function-args ShellScriptByFreqPartHandler.__init__(self, paths, PER_ONCE, **_kwargs) + +# vi: syntax=python ts=4 From 3251001f268ac592b9ed744e945456acc72ed2e2 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Fri, 5 Mar 2021 14:05:34 -0600 Subject: [PATCH 17/49] Fixes for flake8; prematurely pushed the last one --- cloudinit/handlers/__init__.py | 7 ++++--- cloudinit/handlers/shell_script_by_frequency.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 45b6c7105c5..0a0a7112318 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -51,9 +51,10 @@ '#cloud-config-archive': 'text/cloud-config-archive', '#cloud-config-jsonp': 'text/cloud-config-jsonp', '## template: jinja': 'text/jinja2', - # Note: for the next 3 entries, the prefix doesn't matter because these are for - # types that can only be used as part of a MIME message. However, including these - # entries supresses warnings during `cloudinit devel make-mime` + # Note: for the next 3 entries, the prefix doesn't matter because these + # are for types that can only be used as part of a MIME message. However, + # including these entries supresses warnings during `cloudinit devel + # make-mime`, which otherwise would require `--force`. 'text/x-shellscript-per-boot': 'text/x-shellscript-per-boot', 'text/x-shellscript-per-instance': 'text/x-shellscript-per-instance', 'text/x-shellscript-per-once': 'text/x-shellscript-per-once' diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index ed31853466e..05f573bf797 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -26,7 +26,7 @@ def get_script_folder_by_frequency(freq, scripts_dir): def write_script_by_frequency(script_path, payload, frequency, scripts_dir): - """Given a filename, a payload, a frequency, and a scripts folder, write + """Given a filename, a payload, a frequency, and a scripts folder, write the payload to the correct frequency-specific path""" filename = os.path.basename(script_path) filename = util.clean_filename(filename) @@ -35,6 +35,7 @@ def write_script_by_frequency(script_path, payload, frequency, scripts_dir): payload = util.dos2unix(payload) util.write_file(path, payload, 0o700) + class ShellScriptByFreqPartHandler(Handler): """Common base class for the frequency-specific script handlers.""" def __init__(self, paths, freq, **_kwargs): From d8cd589249775a135001e293485ce9c32849ece4 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Fri, 5 Mar 2021 15:59:57 -0600 Subject: [PATCH 18/49] Removed subclasses - now there's just one class with frequency-specific instances added in stages.py --- .../handlers/shell_script_by_frequency.py | 38 +++---------------- cloudinit/stages.py | 15 ++++---- 2 files changed, 13 insertions(+), 40 deletions(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index 05f573bf797..c1c7151a6e9 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -38,7 +38,11 @@ def write_script_by_frequency(script_path, payload, frequency, scripts_dir): class ShellScriptByFreqPartHandler(Handler): """Common base class for the frequency-specific script handlers.""" - def __init__(self, paths, freq, **_kwargs): + prefixes = ["text/x-shellscript-per-boot", + "text/x-shellscript-per-instance", + "text/x-shellscript-per-once"] + + def __init__(self, freq, paths, **_kwargs): Handler.__init__(self, freq) self.scripts_dir = paths.get_cpath('scripts') if 'script_path' in _kwargs: @@ -51,35 +55,3 @@ def handle_part(self, data, ctype, script_path, payload, frequency): filename = util.clean_filename(filename) write_script_by_frequency(script_path, payload, self.frequency, self.scripts_dir) - - -# per-boot -class ShellScriptPerBootPartHandler(ShellScriptByFreqPartHandler): - prefixes = ["text/x-shellscript-per-boot"] - - def __init__(self, paths, **_kwargs): - # pylint: disable=too-many-function-args - ShellScriptByFreqPartHandler.__init__(self, paths, PER_ALWAYS, - **_kwargs) - - -# per-instance -class ShellScriptPerInstancePartHandler(ShellScriptByFreqPartHandler): - prefixes = ["text/x-shellscript-per-instance"] - - def __init__(self, paths, **_kwargs): - # pylint: disable=too-many-function-args - ShellScriptByFreqPartHandler.__init__(self, paths, PER_INSTANCE, - **_kwargs) - - -# per-once -class ShellScriptPerOncePartHandler(ShellScriptByFreqPartHandler): - prefixes = ["text/x-shellscript-per-once"] - - def __init__(self, paths, **_kwargs): - # pylint: disable=too-many-function-args - ShellScriptByFreqPartHandler.__init__(self, paths, PER_ONCE, - **_kwargs) - -# vi: syntax=python ts=4 diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 1cf38081104..8667e0600f9 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -10,7 +10,8 @@ import sys from cloudinit.settings import ( - FREQUENCIES, CLOUD_CONFIG, PER_INSTANCE, RUN_CLOUD_CONFIG) + FREQUENCIES, CLOUD_CONFIG, PER_ALWAYS, PER_INSTANCE, PER_ONCE, + RUN_CLOUD_CONFIG) from cloudinit import handlers @@ -20,9 +21,7 @@ from cloudinit.handlers.jinja_template import JinjaTemplatePartHandler from cloudinit.handlers.shell_script import ShellScriptPartHandler from cloudinit.handlers.shell_script_by_frequency import \ - (ShellScriptPerBootPartHandler, - ShellScriptPerInstancePartHandler, - ShellScriptPerOncePartHandler) + ShellScriptByFreqPartHandler from cloudinit.handlers.upstart_job import UpstartJobPartHandler from cloudinit.event import EventType @@ -419,10 +418,12 @@ def _default_handlers(self, opts=None): # TODO(harlowja) Hmmm, should we dynamically import these?? cloudconfig_handler = CloudConfigPartHandler(**opts) shellscript_handler = ShellScriptPartHandler(**opts) - shellscript_per_boot_handler = ShellScriptPerBootPartHandler(**opts) + shellscript_per_boot_handler = \ + ShellScriptByFreqPartHandler(PER_ALWAYS, **opts) shellscript_per_instance_handler = \ - ShellScriptPerInstancePartHandler(**opts) - shellscript_per_once_handler = ShellScriptPerOncePartHandler(**opts) + ShellScriptByFreqPartHandler(PER_INSTANCE, **opts) + shellscript_per_once_handler = \ + ShellScriptByFreqPartHandler(PER_ONCE, **opts) def_handlers = [ cloudconfig_handler, shellscript_handler, From 45cfa7cdc740096e6dbc0735c3a19c151533ce4f Mon Sep 17 00:00:00 2001 From: beantaxi Date: Wed, 7 Apr 2021 22:55:29 -0500 Subject: [PATCH 19/49] start of integration test ... plus cloudinit-tester scripts for fun --- Dockerfile | 15 ++++++ cloudinit-tester-build.sh | 7 +++ cloudinit-tester-cmds.sh | 20 +++++++ cloudinit-tester-run.sh | 8 +++ cloudinit-tester-shell.sh | 10 ++++ cloudinit/cmd/devel/make_mime.py | 54 +++++++++++-------- .../integration_tests/shellscript-by-freq.py | 36 +++++++++++++ 7 files changed, 127 insertions(+), 23 deletions(-) create mode 100644 Dockerfile create mode 100755 cloudinit-tester-build.sh create mode 100755 cloudinit-tester-cmds.sh create mode 100755 cloudinit-tester-run.sh create mode 100755 cloudinit-tester-shell.sh create mode 100644 tests/integration_tests/shellscript-by-freq.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..8796e78ee01 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3-slim +RUN apt-get update -y +RUN apt-get install -y gcc git +COPY ./requirements.txt /src/ +COPY ./test-requirements.txt /src/ +COPY ./integration-requirements.txt /src/ +WORKDIR /src +RUN pip3 install --upgrade pip +RUN pip3 install --requirement requirements.txt +RUN pip3 install --requirement test-requirements.txt +RUN pip3 install --requirement integration-requirements.txt +RUN pip3 install flake8 pylint +VOLUME /src +ENTRYPOINT ["python", "-m"] + diff --git a/cloudinit-tester-build.sh b/cloudinit-tester-build.sh new file mode 100755 index 00000000000..df3743e17cd --- /dev/null +++ b/cloudinit-tester-build.sh @@ -0,0 +1,7 @@ +#! /usr/bin/env sh + +# Use this script to build the docker image. +# Not too fancy, but it does ensure the image gets a predictable name, +# for use by other scripts. + +docker build --tag cloudinit-tester . \ No newline at end of file diff --git a/cloudinit-tester-cmds.sh b/cloudinit-tester-cmds.sh new file mode 100755 index 00000000000..5e670cd534b --- /dev/null +++ b/cloudinit-tester-cmds.sh @@ -0,0 +1,20 @@ +#! /usr/bin/env sh + +# Use this script to run 3 useful invocations of cloudinit-tester-run.sh +# It's easier to imagine a fancier script that asks Y/n before each test, +# etc etc, but at that point it might make sense to switch to python + +# Flake8 +printf 'Running flake8 ...' +./cloudinit-tester-run.sh flake8 --verbose cloudinit tests/unittests +read -p 'flake8 done (press any key to continue ...)' + +# pylint +printf 'Running pylint (this could take a while) ...' +./cloudinit-tester-run.sh pylint cloudinit tests/unittests +read -p '(press any key to continue ...)' + +# pytest +printf 'Running pytest ...' +./cloudinit-tester-run.sh pytest tests/unittests +read -p '(press any key to continue ...)' \ No newline at end of file diff --git a/cloudinit-tester-run.sh b/cloudinit-tester-run.sh new file mode 100755 index 00000000000..71dbd896e5e --- /dev/null +++ b/cloudinit-tester-run.sh @@ -0,0 +1,8 @@ +# !/usr/bin/env sh + +# Use this script to run various commands inside a cloudinit-tester container. +# The cloudinit-tester container has an ENTRYPOINT of 'python -m', so use the +# script like this: +# +# ./cloud-init-tester.sh pylint cloudinit # This runs python -m pylint cloudinit +docker run --rm --name cloudinit-tester --volume $PWD:/src cloudinit-tester "$@" diff --git a/cloudinit-tester-shell.sh b/cloudinit-tester-shell.sh new file mode 100755 index 00000000000..6a1f442bc45 --- /dev/null +++ b/cloudinit-tester-shell.sh @@ -0,0 +1,10 @@ +#! /usr/bin/env sh + +# Use this script to startup a cloudinit-tester and then shell into it. +# This is potentially useful if you want to run a large number of commands, +# or play around. + +# It also supports cmdline args at the end, to allow for running arbitrary commands +# in case the default entrypoint of `python -m` is not suitable. + +docker run --rm --name cloudinit-tester --volume $PWD:/src -it --entrypoint /bin/bash cloudinit-tester "$@" diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index 4e6a57786ac..c815bdf1294 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -17,6 +17,35 @@ "-a script.sh:x-shellscript > user-data") +def create_mime_message(args): + sub_messages = [] + errors = [] + rc = 0 + for i, (fh, filename, format_type) in enumerate(args.files): + contents = fh.read() + sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) + sub_message.add_header('Content-Disposition', + 'attachment; filename="%s"' % (filename)) + content_type = sub_message.get_content_type().lower() + if content_type not in get_content_types(): + level = "WARNING" if args.force else "ERROR" + msg = (level + ": content type %r for attachment %s " + "may be incorrect!") % (content_type, i + 1) + sys.stderr.write(msg + '\n') + errors.append(msg) + sub_messages.append(sub_message) + if len(errors) and not args.force: + sys.stderr.write("Invalid content-types, override with --force\n") + combined_message = None + rc = 1 + else: + combined_message = MIMEMultipart() + for msg in sub_messages: + combined_message.attach(msg) + rc = 0 + return (combined_message, 1) + + def file_content_type(text): """ Return file content type by reading the first line of the input. """ try: @@ -62,7 +91,6 @@ def get_content_types(strip_prefix=False): return sorted([ctype.replace("text/", "") if strip_prefix else ctype for ctype in INCLUSION_TYPES_MAP.values()]) - def handle_args(name, args): """Create a multi-part MIME archive for use as user-data. Optionally print out the list of supported content types of cloud-init. @@ -77,29 +105,9 @@ def handle_args(name, args): print("\n".join(get_content_types(strip_prefix=True))) return 0 - sub_messages = [] - errors = [] - for i, (fh, filename, format_type) in enumerate(args.files): - contents = fh.read() - sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) - sub_message.add_header('Content-Disposition', - 'attachment; filename="%s"' % (filename)) - content_type = sub_message.get_content_type().lower() - if content_type not in get_content_types(): - level = "WARNING" if args.force else "ERROR" - msg = (level + ": content type %r for attachment %s " - "may be incorrect!") % (content_type, i + 1) - sys.stderr.write(msg + '\n') - errors.append(msg) - sub_messages.append(sub_message) - if len(errors) and not args.force: - sys.stderr.write("Invalid content-types, override with --force\n") - return 1 - combined_message = MIMEMultipart() - for msg in sub_messages: - combined_message.attach(msg) + (combined_message, rc) = create_mime_message(args) print(combined_message) - return 0 + return rc def main(): diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py new file mode 100644 index 00000000000..decf7a54101 --- /dev/null +++ b/tests/integration_tests/shellscript-by-freq.py @@ -0,0 +1,36 @@ + +"""Integration tests for various handlers.""" + +from io import StringIO +from types import SimpleNamespace + +import pytest + +from cloudinit.cmd.devel.make_mime import create_mime_message +from tests.integration_tests.instances import IntegrationInstance + +PER_FREQ_TEMPLATE = """\ +#!/bin/bash +touch /var/tmp/test_per_freq_{} +""" + +PER_BOOT_FILE = StringIO(PER_FREQ_TEMPLATE.format('boot')) +PER_INSTANCE_FILE = StringIO(PER_FREQ_TEMPLATE.format('instance')) +PER_ONCE_FILE = StringIO(PER_FREQ_TEMPLATE.format('once')) + +args = SimpleNamespace( + debug=False, + list_types=False, + files=[ + (PER_BOOT_FILE, 'boot.sh', 'x-shellscript-per-boot'), + (PER_INSTANCE_FILE, 'instance.sh', 'x-shellscript-per-instance'), + (PER_ONCE_FILE, 'once.sh', 'x-shellscript-per-once'), + ] +) + +USER_DATA = create_mime_message(args) + +@pytest.mark.user_data(USER_DATA) +def test_per_freq(client: IntegrationInstance): + scripts = client.execute('find /var/lib/cloud/scripts') + print(scripts) \ No newline at end of file From bee79ad94c1bfe8ce484d50371f6ec03195b72ea Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Fri, 31 Dec 2021 20:48:43 -0600 Subject: [PATCH 20/49] Still trying to get integration test working --- cloudinit/cmd/devel/make_mime.py | 2 +- cloudinit/handlers/shell_script_by_frequency.py | 4 ++-- tests/integration_tests/shellscript-by-freq.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index c815bdf1294..1097d4b9403 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -43,7 +43,7 @@ def create_mime_message(args): for msg in sub_messages: combined_message.attach(msg) rc = 0 - return (combined_message, 1) + return (combined_message, rc) def file_content_type(text): diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index c1c7151a6e9..0775b988d36 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -5,8 +5,8 @@ from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE LOG = log.getLogger(__name__) -# cloutinit/settings.py defines PER_*** frequency constants. It makes sense to -# use them here, instead hardcodes, and map them to the 'per-***' frequency- +# cloudinit/settings.py defines PER_*** frequency constants. It makes sense to +# use them here, instead of hardcodes, and map them to the 'per-***' frequency- # specific folders in /v/l/c/scripts. It might make sense to expose this at a # higher level or in a more general module -- eg maybe in cloudinit/settings.py # itself -- but for now it's here. diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index decf7a54101..62eb2894bb8 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -28,9 +28,9 @@ ] ) -USER_DATA = create_mime_message(args) +USER_DATA, errors = create_mime_message(args) @pytest.mark.user_data(USER_DATA) def test_per_freq(client: IntegrationInstance): - scripts = client.execute('find /var/lib/cloud/scripts') + scripts = client.execute('find /var/lib/cloud/scripts -type f') print(scripts) \ No newline at end of file From 1e58a42d0a1938d737c33fc48c4d05df23587a66 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Fri, 31 Dec 2021 22:04:18 -0600 Subject: [PATCH 21/49] Trying to get integration test working ... --- cloudinit/cmd/devel/make_mime.py | 19 ++++++++++++------- conftest.py | 4 ++-- tests/integration_tests/conftest.py | 8 ++++---- .../integration_tests/shellscript-by-freq.py | 7 ++++--- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index 1097d4b9403..311c75dc96d 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -31,19 +31,17 @@ def create_mime_message(args): level = "WARNING" if args.force else "ERROR" msg = (level + ": content type %r for attachment %s " "may be incorrect!") % (content_type, i + 1) - sys.stderr.write(msg + '\n') + # sys.stderr.write(msg + '\n') errors.append(msg) sub_messages.append(sub_message) if len(errors) and not args.force: - sys.stderr.write("Invalid content-types, override with --force\n") + # sys.stderr.write("Invalid content-types, override with --force\n") combined_message = None - rc = 1 else: combined_message = MIMEMultipart() for msg in sub_messages: combined_message.attach(msg) - rc = 0 - return (combined_message, rc) + return (combined_message, errors) def file_content_type(text): @@ -105,9 +103,16 @@ def handle_args(name, args): print("\n".join(get_content_types(strip_prefix=True))) return 0 - (combined_message, rc) = create_mime_message(args) + combined_message, errors = create_mime_message(args) + if errors: + level = 'WARNING' if args.force else 'ERROR' + for error in errors: + sys.stderr.write(f'{level}: {error}\n') + sys.stderr.write("Invalid content-types, override with --force\n") + if not args.force: + return 1 print(combined_message) - return rc + return 0 def main(): diff --git a/conftest.py b/conftest.py index 9e9d9ff8dd5..97d06defbca 100644 --- a/conftest.py +++ b/conftest.py @@ -65,7 +65,7 @@ def closest_marker_first_arg_or(request, marker_name: str, default): return result[0] -@pytest.yield_fixture(autouse=True) +@pytest.fixture(autouse=True) def disable_subp_usage(request, fixture_utils): """ Across all (pytest) tests, ensure that subp.subp is not invoked. @@ -165,7 +165,7 @@ def fixture_utils(): return _FixtureUtils -@pytest.yield_fixture +@pytest.fixture def httpretty(): """ Enable HTTPretty for duration of the testcase, resetting before and after. diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 0b93c94f3f0..bd7c3e1c779 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -90,7 +90,7 @@ def disable_subp_usage(request): pass -@pytest.yield_fixture(scope='session') +@pytest.fixture(scope='session') def session_cloud(): if integration_settings.PLATFORM not in platforms.keys(): raise ValueError( @@ -231,21 +231,21 @@ def _client(request, fixture_utils, session_cloud: IntegrationCloud): _collect_logs(instance, request.node.nodeid, test_failed) -@pytest.yield_fixture +@pytest.fixture def client(request, fixture_utils, session_cloud, setup_image): """Provide a client that runs for every test.""" with _client(request, fixture_utils, session_cloud) as client: yield client -@pytest.yield_fixture(scope='module') +@pytest.fixture(scope='module') def module_client(request, fixture_utils, session_cloud, setup_image): """Provide a client that runs once per module.""" with _client(request, fixture_utils, session_cloud) as client: yield client -@pytest.yield_fixture(scope='class') +@pytest.fixture(scope='class') def class_client(request, fixture_utils, session_cloud, setup_image): """Provide a client that runs once per class.""" with _client(request, fixture_utils, session_cloud) as client: diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index 62eb2894bb8..8fc2acce2cb 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -30,7 +30,8 @@ USER_DATA, errors = create_mime_message(args) -@pytest.mark.user_data(USER_DATA) +# @pytest.mark.user_data(USER_DATA) def test_per_freq(client: IntegrationInstance): - scripts = client.execute('find /var/lib/cloud/scripts -type f') - print(scripts) \ No newline at end of file + pass + # scripts = client.execute('find /var/lib/cloud/scripts -type f') + # print(scripts) \ No newline at end of file From 0a1bdc8c2934b0740b888c50543b4f25f0de8591 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 1 Jan 2022 04:11:21 +0000 Subject: [PATCH 22/49] dunno --- tests/integration_tests/shellscript-by-freq.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index decf7a54101..9cb2053bd7c 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -28,9 +28,13 @@ ] ) -USER_DATA = create_mime_message(args) +(USER_DATA, rc) = create_mime_message(args) @pytest.mark.user_data(USER_DATA) def test_per_freq(client: IntegrationInstance): - scripts = client.execute('find /var/lib/cloud/scripts') - print(scripts) \ No newline at end of file + rc = client.execute('test -d /var/lib/cloud/scripts').ok + assert rc is True + rc = client.execute('test -f /var/tmp/test_per_freq_boot').ok + assert rc is True + rc = client.execute('test -f /var/lib/cloud/scripts/per-boot/boot.sh').ok + assert rc is True From 400ac42f95853fa2abc7483ad145235b9f58e4ae Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Sat, 1 Jan 2022 09:06:33 -0600 Subject: [PATCH 23/49] added print statements --- tests/integration_tests/shellscript-by-freq.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index bdc8ccd4789..a2930fd93c6 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -32,10 +32,13 @@ # @pytest.mark.user_data(USER_DATA) def test_per_freq(client: IntegrationInstance): + print('checking /v/l/c/scripts exists ...') rc = client.execute('test -d /var/lib/cloud/scripts').ok assert rc is True + print('checking /v/tmp/c/test_per_freq_boot exists ...') rc = client.execute('test -f /var/tmp/test_per_freq_boot').ok assert rc is True + print('checking /v/l/c/s/per-boot/boot.sh exists ...') rc = client.execute('test -f /var/lib/cloud/scripts/per-boot/boot.sh').ok assert rc is True pass From 0671d0865fa7806e89fe2742d0cbef5d3cd2ca5b Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Sat, 1 Jan 2022 09:15:56 -0600 Subject: [PATCH 24/49] Added print() statements for debugging --- cloudinit/handlers/shell_script_by_frequency.py | 4 ++++ tests/integration_tests/shellscript-by-freq.py | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index 0775b988d36..a503a49a3b7 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -28,11 +28,13 @@ def get_script_folder_by_frequency(freq, scripts_dir): def write_script_by_frequency(script_path, payload, frequency, scripts_dir): """Given a filename, a payload, a frequency, and a scripts folder, write the payload to the correct frequency-specific path""" + print('write_script_by_frequency()') filename = os.path.basename(script_path) filename = util.clean_filename(filename) folder = get_script_folder_by_frequency(frequency, scripts_dir) path = os.path.join(folder, filename) payload = util.dos2unix(payload) + print(f'Writing payload to {path}') util.write_file(path, payload, 0o700) @@ -43,12 +45,14 @@ class ShellScriptByFreqPartHandler(Handler): "text/x-shellscript-per-once"] def __init__(self, freq, paths, **_kwargs): + print("ShellScriptByFreqPartHandler: c'tor()") Handler.__init__(self, freq) self.scripts_dir = paths.get_cpath('scripts') if 'script_path' in _kwargs: self.scripts_dir = paths.get_cpath(_kwargs['script_path']) def handle_part(self, data, ctype, script_path, payload, frequency): + print("ShellScriptByFreqPartHandler.handle_part()") if script_path is not None: LOG.debug("script_path=%s", script_path) filename = os.path.basename(script_path) diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index a2930fd93c6..e4a8d856a8a 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -35,10 +35,10 @@ def test_per_freq(client: IntegrationInstance): print('checking /v/l/c/scripts exists ...') rc = client.execute('test -d /var/lib/cloud/scripts').ok assert rc is True - print('checking /v/tmp/c/test_per_freq_boot exists ...') - rc = client.execute('test -f /var/tmp/test_per_freq_boot').ok - assert rc is True print('checking /v/l/c/s/per-boot/boot.sh exists ...') rc = client.execute('test -f /var/lib/cloud/scripts/per-boot/boot.sh').ok assert rc is True + print('checking /v/tmp/c/test_per_freq_boot exists ...') + rc = client.execute('test -f /var/tmp/test_per_freq_boot').ok + assert rc is True pass From 74bf2fc9058dfabeaff73a1e3fcefa5f69cf770a Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Sat, 1 Jan 2022 09:42:16 -0600 Subject: [PATCH 25/49] second pass at incorporating falcons changes --- cloudinit/cmd/devel/make_mime.py | 25 ++++------ .../handlers/shell_script_by_frequency.py | 2 +- .../integration_tests/shellscript-by-freq.py | 46 +++++++++++-------- 3 files changed, 36 insertions(+), 37 deletions(-) diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index 311c75dc96d..4f5de3504cb 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -17,30 +17,23 @@ "-a script.sh:x-shellscript > user-data") -def create_mime_message(args): +def create_mime_message(files): sub_messages = [] errors = [] - rc = 0 - for i, (fh, filename, format_type) in enumerate(args.files): + for i, (fh, filename, format_type) in enumerate(files): contents = fh.read() sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) sub_message.add_header('Content-Disposition', 'attachment; filename="%s"' % (filename)) content_type = sub_message.get_content_type().lower() - if content_type not in get_content_types(): - level = "WARNING" if args.force else "ERROR" - msg = (level + ": content type %r for attachment %s " - "may be incorrect!") % (content_type, i + 1) - # sys.stderr.write(msg + '\n') + if content_type not in get_content_types(): + msg = ("content type %r for attachment %s " + "may be incorrect!") % (content_type, i+1) errors.append(msg) sub_messages.append(sub_message) - if len(errors) and not args.force: - # sys.stderr.write("Invalid content-types, override with --force\n") - combined_message = None - else: - combined_message = MIMEMultipart() - for msg in sub_messages: - combined_message.attach(msg) + combined_message = MIMEMultipart() + for msg in sub_messages: + combined_message.attach(msg) return (combined_message, errors) @@ -103,7 +96,7 @@ def handle_args(name, args): print("\n".join(get_content_types(strip_prefix=True))) return 0 - combined_message, errors = create_mime_message(args) + combined_message, errors = create_mime_message(args.files) if errors: level = 'WARNING' if args.force else 'ERROR' for error in errors: diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index a503a49a3b7..0ff6840af55 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -53,8 +53,8 @@ def __init__(self, freq, paths, **_kwargs): def handle_part(self, data, ctype, script_path, payload, frequency): print("ShellScriptByFreqPartHandler.handle_part()") + LOG.debug("script_path=%s", script_path) if script_path is not None: - LOG.debug("script_path=%s", script_path) filename = os.path.basename(script_path) filename = util.clean_filename(filename) write_script_by_frequency(script_path, payload, self.frequency, diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index e4a8d856a8a..a8b323ccb91 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -11,34 +11,40 @@ PER_FREQ_TEMPLATE = """\ #!/bin/bash -touch /var/tmp/test_per_freq_{} +touch /tmp/test_per_freq_{} """ PER_BOOT_FILE = StringIO(PER_FREQ_TEMPLATE.format('boot')) PER_INSTANCE_FILE = StringIO(PER_FREQ_TEMPLATE.format('instance')) PER_ONCE_FILE = StringIO(PER_FREQ_TEMPLATE.format('once')) -args = SimpleNamespace( - debug=False, - list_types=False, - files=[ - (PER_BOOT_FILE, 'boot.sh', 'x-shellscript-per-boot'), - (PER_INSTANCE_FILE, 'instance.sh', 'x-shellscript-per-instance'), - (PER_ONCE_FILE, 'once.sh', 'x-shellscript-per-once'), - ] -) - -USER_DATA, errors = create_mime_message(args) - -# @pytest.mark.user_data(USER_DATA) +# args = SimpleNamespace( +# debug=False, +# list_types=False, +# files=[ +# (PER_BOOT_FILE, 'boot.sh', 'x-shellscript-per-boot'), +# (PER_INSTANCE_FILE, 'instance.sh', 'x-shellscript-per-instance'), +# (PER_ONCE_FILE, 'once.sh', 'x-shellscript-per-once'), +# ] +# ) + +FILES=[ + (PER_BOOT_FILE, 'boot.sh', 'x-shellscript-per-boot'), + (PER_INSTANCE_FILE, 'instance.sh', 'x-shellscript-per-instance'), + (PER_ONCE_FILE, 'once.sh', 'x-shellscript-per-once'), +] + +USER_DATA, errors = create_mime_message(FILES) + +@pytest.mark.user_data(USER_DATA) def test_per_freq(client: IntegrationInstance): print('checking /v/l/c/scripts exists ...') - rc = client.execute('test -d /var/lib/cloud/scripts').ok - assert rc is True + rc_ok = client.execute('test -d /var/lib/cloud/scripts').ok + assert rc_ok is True print('checking /v/l/c/s/per-boot/boot.sh exists ...') - rc = client.execute('test -f /var/lib/cloud/scripts/per-boot/boot.sh').ok - assert rc is True + rc_ok = client.execute('test -f /var/lib/cloud/scripts/per-boot/boot.sh').ok + assert rc_ok is True print('checking /v/tmp/c/test_per_freq_boot exists ...') - rc = client.execute('test -f /var/tmp/test_per_freq_boot').ok - assert rc is True + rc_ok = client.execute('test -f /var/tmp/test_per_freq_boot').ok + assert rc_ok is True pass From b56c2f850d72125780f9742665eece49978d13f2 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Sat, 1 Jan 2022 09:51:33 -0600 Subject: [PATCH 26/49] more logging --- .gitignore | 1 + cloudinit/cmd/devel/make_mime.py | 4 ++++ tests/integration_tests/shellscript-by-freq.py | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index eb26e0da91e..121c668be63 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ cloud-init_*.upload # user test settings tests/integration_tests/user_settings.py +misc/ diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index 4f5de3504cb..ac1b9b87531 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -18,9 +18,12 @@ def create_mime_message(files): + print(f'files={files}') + print(f'len(files)={len(files)}') sub_messages = [] errors = [] for i, (fh, filename, format_type) in enumerate(files): + print(f'files[{i}]={filename}') contents = fh.read() sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) sub_message.add_header('Content-Disposition', @@ -33,6 +36,7 @@ def create_mime_message(files): sub_messages.append(sub_message) combined_message = MIMEMultipart() for msg in sub_messages: + print('attaching message') combined_message.attach(msg) return (combined_message, errors) diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index a8b323ccb91..79c1473d336 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -35,7 +35,8 @@ ] USER_DATA, errors = create_mime_message(FILES) - +print(f'errors={errors}') +print(f'USER_DATA=${USER_DATA}') @pytest.mark.user_data(USER_DATA) def test_per_freq(client: IntegrationInstance): print('checking /v/l/c/scripts exists ...') From 69bf96409a56d7accaec24e3aa483cc8203c6117 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Sat, 1 Jan 2022 15:52:34 +0000 Subject: [PATCH 27/49] dunno --- cloudinit/handlers/shell_script_by_frequency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index a503a49a3b7..8151a98b33d 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -53,8 +53,8 @@ def __init__(self, freq, paths, **_kwargs): def handle_part(self, data, ctype, script_path, payload, frequency): print("ShellScriptByFreqPartHandler.handle_part()") + print("script_path=%s", script_path) if script_path is not None: - LOG.debug("script_path=%s", script_path) filename = os.path.basename(script_path) filename = util.clean_filename(filename) write_script_by_frequency(script_path, payload, self.frequency, From 09a3006096790ec2677d10607373bcc4eaa804ca Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Sat, 1 Jan 2022 10:01:25 -0600 Subject: [PATCH 28/49] added more tests --- .../integration_tests/shellscript-by-freq.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index 79c1473d336..c88bb24b6e5 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -42,10 +42,25 @@ def test_per_freq(client: IntegrationInstance): print('checking /v/l/c/scripts exists ...') rc_ok = client.execute('test -d /var/lib/cloud/scripts').ok assert rc_ok is True + # Test per-boot print('checking /v/l/c/s/per-boot/boot.sh exists ...') rc_ok = client.execute('test -f /var/lib/cloud/scripts/per-boot/boot.sh').ok assert rc_ok is True - print('checking /v/tmp/c/test_per_freq_boot exists ...') - rc_ok = client.execute('test -f /var/tmp/test_per_freq_boot').ok + print('checking /tmp/c/test_per_freq_boot exists ...') + rc_ok = client.execute('test -f /tmp/test_per_freq_boot').ok + assert rc_ok is True + # Test per-instance + print('checking /v/l/c/s/per-boot/instance.sh exists ...') + rc_ok = client.execute('test -f /var/lib/cloud/scripts/per-boot/instance.sh').ok + assert rc_ok is True + print('checking /tmp/c/test_per_freq_instance exists ...') + rc_ok = client.execute('test -f /tmp/test_per_freq_instance').ok + assert rc_ok is True + # Test per-once + print('checking /v/l/c/s/per-boot/once.sh exists ...') + rc_ok = client.execute('test -f /var/lib/cloud/scripts/per-boot/once.sh').ok + assert rc_ok is True + print('checking /tmp/c/test_per_freq_once exists ...') + rc_ok = client.execute('test -f /tmp/test_per_freq_once').ok assert rc_ok is True pass From 2d32b5372f3c93a378e01ee6f744596fca79a0a4 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Mon, 3 Jan 2022 09:22:49 -0600 Subject: [PATCH 29/49] moved Docker test stuff to cloudinit/cmd/devel --- Dockerfile => cloudinit/cmd/devel/Dockerfile | 0 .../cmd/devel/cloudinit-tester-build.sh | 0 .../cmd/devel/cloudinit-tester-cmds.sh | 0 .../cmd/devel/cloudinit-tester-run.sh | 0 .../cmd/devel/cloudinit-tester-shell.sh | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename Dockerfile => cloudinit/cmd/devel/Dockerfile (100%) rename cloudinit-tester-build.sh => cloudinit/cmd/devel/cloudinit-tester-build.sh (100%) rename cloudinit-tester-cmds.sh => cloudinit/cmd/devel/cloudinit-tester-cmds.sh (100%) rename cloudinit-tester-run.sh => cloudinit/cmd/devel/cloudinit-tester-run.sh (100%) rename cloudinit-tester-shell.sh => cloudinit/cmd/devel/cloudinit-tester-shell.sh (100%) diff --git a/Dockerfile b/cloudinit/cmd/devel/Dockerfile similarity index 100% rename from Dockerfile rename to cloudinit/cmd/devel/Dockerfile diff --git a/cloudinit-tester-build.sh b/cloudinit/cmd/devel/cloudinit-tester-build.sh similarity index 100% rename from cloudinit-tester-build.sh rename to cloudinit/cmd/devel/cloudinit-tester-build.sh diff --git a/cloudinit-tester-cmds.sh b/cloudinit/cmd/devel/cloudinit-tester-cmds.sh similarity index 100% rename from cloudinit-tester-cmds.sh rename to cloudinit/cmd/devel/cloudinit-tester-cmds.sh diff --git a/cloudinit-tester-run.sh b/cloudinit/cmd/devel/cloudinit-tester-run.sh similarity index 100% rename from cloudinit-tester-run.sh rename to cloudinit/cmd/devel/cloudinit-tester-run.sh diff --git a/cloudinit-tester-shell.sh b/cloudinit/cmd/devel/cloudinit-tester-shell.sh similarity index 100% rename from cloudinit-tester-shell.sh rename to cloudinit/cmd/devel/cloudinit-tester-shell.sh From 776a51d4414145fc0d1b06b7760ff665ccd3a449 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Mon, 3 Jan 2022 09:26:07 -0600 Subject: [PATCH 30/49] removed Dockerfiles from this branch - they will be added to another feature branch specific to their PR --- cloudinit/cmd/devel/Dockerfile | 15 -------------- cloudinit/cmd/devel/cloudinit-tester-build.sh | 7 ------- cloudinit/cmd/devel/cloudinit-tester-cmds.sh | 20 ------------------- cloudinit/cmd/devel/cloudinit-tester-run.sh | 8 -------- cloudinit/cmd/devel/cloudinit-tester-shell.sh | 10 ---------- 5 files changed, 60 deletions(-) delete mode 100644 cloudinit/cmd/devel/Dockerfile delete mode 100755 cloudinit/cmd/devel/cloudinit-tester-build.sh delete mode 100755 cloudinit/cmd/devel/cloudinit-tester-cmds.sh delete mode 100755 cloudinit/cmd/devel/cloudinit-tester-run.sh delete mode 100755 cloudinit/cmd/devel/cloudinit-tester-shell.sh diff --git a/cloudinit/cmd/devel/Dockerfile b/cloudinit/cmd/devel/Dockerfile deleted file mode 100644 index 8796e78ee01..00000000000 --- a/cloudinit/cmd/devel/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM python:3-slim -RUN apt-get update -y -RUN apt-get install -y gcc git -COPY ./requirements.txt /src/ -COPY ./test-requirements.txt /src/ -COPY ./integration-requirements.txt /src/ -WORKDIR /src -RUN pip3 install --upgrade pip -RUN pip3 install --requirement requirements.txt -RUN pip3 install --requirement test-requirements.txt -RUN pip3 install --requirement integration-requirements.txt -RUN pip3 install flake8 pylint -VOLUME /src -ENTRYPOINT ["python", "-m"] - diff --git a/cloudinit/cmd/devel/cloudinit-tester-build.sh b/cloudinit/cmd/devel/cloudinit-tester-build.sh deleted file mode 100755 index df3743e17cd..00000000000 --- a/cloudinit/cmd/devel/cloudinit-tester-build.sh +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/env sh - -# Use this script to build the docker image. -# Not too fancy, but it does ensure the image gets a predictable name, -# for use by other scripts. - -docker build --tag cloudinit-tester . \ No newline at end of file diff --git a/cloudinit/cmd/devel/cloudinit-tester-cmds.sh b/cloudinit/cmd/devel/cloudinit-tester-cmds.sh deleted file mode 100755 index 5e670cd534b..00000000000 --- a/cloudinit/cmd/devel/cloudinit-tester-cmds.sh +++ /dev/null @@ -1,20 +0,0 @@ -#! /usr/bin/env sh - -# Use this script to run 3 useful invocations of cloudinit-tester-run.sh -# It's easier to imagine a fancier script that asks Y/n before each test, -# etc etc, but at that point it might make sense to switch to python - -# Flake8 -printf 'Running flake8 ...' -./cloudinit-tester-run.sh flake8 --verbose cloudinit tests/unittests -read -p 'flake8 done (press any key to continue ...)' - -# pylint -printf 'Running pylint (this could take a while) ...' -./cloudinit-tester-run.sh pylint cloudinit tests/unittests -read -p '(press any key to continue ...)' - -# pytest -printf 'Running pytest ...' -./cloudinit-tester-run.sh pytest tests/unittests -read -p '(press any key to continue ...)' \ No newline at end of file diff --git a/cloudinit/cmd/devel/cloudinit-tester-run.sh b/cloudinit/cmd/devel/cloudinit-tester-run.sh deleted file mode 100755 index 71dbd896e5e..00000000000 --- a/cloudinit/cmd/devel/cloudinit-tester-run.sh +++ /dev/null @@ -1,8 +0,0 @@ -# !/usr/bin/env sh - -# Use this script to run various commands inside a cloudinit-tester container. -# The cloudinit-tester container has an ENTRYPOINT of 'python -m', so use the -# script like this: -# -# ./cloud-init-tester.sh pylint cloudinit # This runs python -m pylint cloudinit -docker run --rm --name cloudinit-tester --volume $PWD:/src cloudinit-tester "$@" diff --git a/cloudinit/cmd/devel/cloudinit-tester-shell.sh b/cloudinit/cmd/devel/cloudinit-tester-shell.sh deleted file mode 100755 index 6a1f442bc45..00000000000 --- a/cloudinit/cmd/devel/cloudinit-tester-shell.sh +++ /dev/null @@ -1,10 +0,0 @@ -#! /usr/bin/env sh - -# Use this script to startup a cloudinit-tester and then shell into it. -# This is potentially useful if you want to run a large number of commands, -# or play around. - -# It also supports cmdline args at the end, to allow for running arbitrary commands -# in case the default entrypoint of `python -m` is not suitable. - -docker run --rm --name cloudinit-tester --volume $PWD:/src -it --entrypoint /bin/bash cloudinit-tester "$@" From ea2a2622719d7c72343eb118c29fce9fa6e4f520 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Thu, 20 Jan 2022 19:16:43 -0600 Subject: [PATCH 31/49] passes tox -e flake8 --- cloudinit/cmd/devel/make_mime.py | 5 ++-- .../integration_tests/shellscript-by-freq.py | 24 ++++++++++++------- tests/unittests/test_builtin_handlers.py | 4 ++-- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index 2398f05cd6d..d879ca9b68d 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -32,9 +32,9 @@ def create_mime_message(files): sub_message.add_header('Content-Disposition', 'attachment; filename="%s"' % (filename)) content_type = sub_message.get_content_type().lower() - if content_type not in get_content_types(): + if content_type not in get_content_types(): msg = ("content type %r for attachment %s " - "may be incorrect!") % (content_type, i+1) + "may be incorrect!") % (content_type, i + 1) errors.append(msg) sub_messages.append(sub_message) combined_message = MIMEMultipart() @@ -106,6 +106,7 @@ def get_content_types(strip_prefix=False): ] ) + def handle_args(name, args): """Create a multi-part MIME archive for use as user-data. Optionally print out the list of supported content types of cloud-init. diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index c88bb24b6e5..6b20828f3a6 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -2,7 +2,7 @@ """Integration tests for various handlers.""" from io import StringIO -from types import SimpleNamespace +# from types import SimpleNamespace import pytest @@ -28,7 +28,7 @@ # ] # ) -FILES=[ +FILES = [ (PER_BOOT_FILE, 'boot.sh', 'x-shellscript-per-boot'), (PER_INSTANCE_FILE, 'instance.sh', 'x-shellscript-per-instance'), (PER_ONCE_FILE, 'once.sh', 'x-shellscript-per-once'), @@ -37,6 +37,8 @@ USER_DATA, errors = create_mime_message(FILES) print(f'errors={errors}') print(f'USER_DATA=${USER_DATA}') + + @pytest.mark.user_data(USER_DATA) def test_per_freq(client: IntegrationInstance): print('checking /v/l/c/scripts exists ...') @@ -44,23 +46,29 @@ def test_per_freq(client: IntegrationInstance): assert rc_ok is True # Test per-boot print('checking /v/l/c/s/per-boot/boot.sh exists ...') - rc_ok = client.execute('test -f /var/lib/cloud/scripts/per-boot/boot.sh').ok + cmd = 'test -f /var/lib/cloud/scripts/per-boot/boot.sh' + rc_ok = client.execute(cmd).ok assert rc_ok is True print('checking /tmp/c/test_per_freq_boot exists ...') - rc_ok = client.execute('test -f /tmp/test_per_freq_boot').ok + cmd = 'test -f /tmp/test_per_freq_boot' + rc_ok = client.execute(cmd).ok assert rc_ok is True # Test per-instance print('checking /v/l/c/s/per-boot/instance.sh exists ...') - rc_ok = client.execute('test -f /var/lib/cloud/scripts/per-boot/instance.sh').ok + cmd = 'test -f /var/lib/cloud/scripts/per-boot/instance.sh' + rc_ok = client.execute(cmd).ok assert rc_ok is True print('checking /tmp/c/test_per_freq_instance exists ...') - rc_ok = client.execute('test -f /tmp/test_per_freq_instance').ok + cmd = 'test -f /tmp/test_per_freq_instance' + rc_ok = client.execute(cmd).ok assert rc_ok is True # Test per-once print('checking /v/l/c/s/per-boot/once.sh exists ...') - rc_ok = client.execute('test -f /var/lib/cloud/scripts/per-boot/once.sh').ok + cmd = 'test -f /var/lib/cloud/scripts/per-boot/once.sh' + rc_ok = client.execute(cmd).ok assert rc_ok is True print('checking /tmp/c/test_per_freq_once exists ...') - rc_ok = client.execute('test -f /tmp/test_per_freq_once').ok + cmd = 'test -f /tmp/test_per_freq_once' + rc_ok = client.execute(cmd).ok assert rc_ok is True pass diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 0ced479693b..0b106fd8246 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -20,14 +20,14 @@ ) from cloudinit.handlers.shell_script import ShellScriptPartHandler from cloudinit.handlers.upstart_job import UpstartJobPartHandler -from cloudinit.settings import PER_ALWAYS, PER_INSTANCE +# from cloudinit.settings import PER_ALWAYS, PER_INSTANCE +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, PER_ONCE) from tests.unittests.helpers import ( CiTestCase, FilesystemMockingTestCase, mock, skipUnlessJinja, ) -from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, PER_ONCE) INSTANCE_DATA_FILE = "instance-data-sensitive.json" From 325cb003303d13174142e9b360b106f8ad78c8a2 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Thu, 20 Jan 2022 19:18:27 -0600 Subject: [PATCH 32/49] passes tox -e format --- cloudinit/cmd/devel/make_mime.py | 22 +++++---- cloudinit/handlers/__init__.py | 26 +++++----- .../handlers/shell_script_by_frequency.py | 39 ++++++++------- cloudinit/stages.py | 21 +++++--- tests/integration_tests/conftest.py | 6 +-- .../integration_tests/shellscript-by-freq.py | 49 ++++++++++--------- tests/unittests/test_builtin_handlers.py | 13 +++-- 7 files changed, 96 insertions(+), 80 deletions(-) diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index d879ca9b68d..3a0181fbe60 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -21,25 +21,27 @@ def create_mime_message(files): - print(f'files={files}') - print(f'len(files)={len(files)}') + print(f"files={files}") + print(f"len(files)={len(files)}") sub_messages = [] errors = [] for i, (fh, filename, format_type) in enumerate(files): - print(f'files[{i}]={filename}') + print(f"files[{i}]={filename}") contents = fh.read() sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) - sub_message.add_header('Content-Disposition', - 'attachment; filename="%s"' % (filename)) + sub_message.add_header( + "Content-Disposition", 'attachment; filename="%s"' % (filename) + ) content_type = sub_message.get_content_type().lower() if content_type not in get_content_types(): - msg = ("content type %r for attachment %s " - "may be incorrect!") % (content_type, i + 1) + msg = ( + "content type %r for attachment %s " "may be incorrect!" + ) % (content_type, i + 1) errors.append(msg) sub_messages.append(sub_message) combined_message = MIMEMultipart() for msg in sub_messages: - print('attaching message') + print("attaching message") combined_message.attach(msg) return (combined_message, errors) @@ -123,9 +125,9 @@ def handle_args(name, args): combined_message, errors = create_mime_message(args.files) if errors: - level = 'WARNING' if args.force else 'ERROR' + level = "WARNING" if args.force else "ERROR" for error in errors: - sys.stderr.write(f'{level}: {error}\n') + sys.stderr.write(f"{level}: {error}\n") sys.stderr.write("Invalid content-types, override with --force\n") if not args.force: return 1 diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 85e0c6a120f..7d8a9208ce2 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -40,23 +40,23 @@ # Different file beginnings to their content type INCLUSION_TYPES_MAP = { - '#include': 'text/x-include-url', - '#include-once': 'text/x-include-once-url', - '#!': 'text/x-shellscript', - '#cloud-config': 'text/cloud-config', - '#upstart-job': 'text/upstart-job', - '#part-handler': 'text/part-handler', - '#cloud-boothook': 'text/cloud-boothook', - '#cloud-config-archive': 'text/cloud-config-archive', - '#cloud-config-jsonp': 'text/cloud-config-jsonp', - '## template: jinja': 'text/jinja2', + "#include": "text/x-include-url", + "#include-once": "text/x-include-once-url", + "#!": "text/x-shellscript", + "#cloud-config": "text/cloud-config", + "#upstart-job": "text/upstart-job", + "#part-handler": "text/part-handler", + "#cloud-boothook": "text/cloud-boothook", + "#cloud-config-archive": "text/cloud-config-archive", + "#cloud-config-jsonp": "text/cloud-config-jsonp", + "## template: jinja": "text/jinja2", # Note: for the next 3 entries, the prefix doesn't matter because these # are for types that can only be used as part of a MIME message. However, # including these entries supresses warnings during `cloudinit devel # make-mime`, which otherwise would require `--force`. - 'text/x-shellscript-per-boot': 'text/x-shellscript-per-boot', - 'text/x-shellscript-per-instance': 'text/x-shellscript-per-instance', - 'text/x-shellscript-per-once': 'text/x-shellscript-per-once' + "text/x-shellscript-per-boot": "text/x-shellscript-per-boot", + "text/x-shellscript-per-instance": "text/x-shellscript-per-instance", + "text/x-shellscript-per-once": "text/x-shellscript-per-once", } # Sorted longest first diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index 8151a98b33d..71d8031d5c3 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -1,8 +1,9 @@ import os -from cloudinit import log -from cloudinit import util + +from cloudinit import log, util from cloudinit.handlers import Handler from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE + LOG = log.getLogger(__name__) # cloudinit/settings.py defines PER_*** frequency constants. It makes sense to @@ -11,15 +12,15 @@ # higher level or in a more general module -- eg maybe in cloudinit/settings.py # itself -- but for now it's here. pathMap = { - PER_ALWAYS: 'per-boot', - PER_INSTANCE: 'per-instance', - PER_ONCE: 'per-once' + PER_ALWAYS: "per-boot", + PER_INSTANCE: "per-instance", + PER_ONCE: "per-once", } def get_script_folder_by_frequency(freq, scripts_dir): """Return the frequency-specific subfolder for a given frequency constant - and parent folder.""" + and parent folder.""" freqPath = pathMap[freq] folder = os.path.join(scripts_dir, freqPath) return folder @@ -27,29 +28,32 @@ def get_script_folder_by_frequency(freq, scripts_dir): def write_script_by_frequency(script_path, payload, frequency, scripts_dir): """Given a filename, a payload, a frequency, and a scripts folder, write - the payload to the correct frequency-specific path""" - print('write_script_by_frequency()') + the payload to the correct frequency-specific path""" + print("write_script_by_frequency()") filename = os.path.basename(script_path) filename = util.clean_filename(filename) folder = get_script_folder_by_frequency(frequency, scripts_dir) path = os.path.join(folder, filename) payload = util.dos2unix(payload) - print(f'Writing payload to {path}') + print(f"Writing payload to {path}") util.write_file(path, payload, 0o700) class ShellScriptByFreqPartHandler(Handler): """Common base class for the frequency-specific script handlers.""" - prefixes = ["text/x-shellscript-per-boot", - "text/x-shellscript-per-instance", - "text/x-shellscript-per-once"] + + prefixes = [ + "text/x-shellscript-per-boot", + "text/x-shellscript-per-instance", + "text/x-shellscript-per-once", + ] def __init__(self, freq, paths, **_kwargs): print("ShellScriptByFreqPartHandler: c'tor()") Handler.__init__(self, freq) - self.scripts_dir = paths.get_cpath('scripts') - if 'script_path' in _kwargs: - self.scripts_dir = paths.get_cpath(_kwargs['script_path']) + self.scripts_dir = paths.get_cpath("scripts") + if "script_path" in _kwargs: + self.scripts_dir = paths.get_cpath(_kwargs["script_path"]) def handle_part(self, data, ctype, script_path, payload, frequency): print("ShellScriptByFreqPartHandler.handle_part()") @@ -57,5 +61,6 @@ def handle_part(self, data, ctype, script_path, payload, frequency): if script_path is not None: filename = os.path.basename(script_path) filename = util.clean_filename(filename) - write_script_by_frequency(script_path, payload, self.frequency, - self.scripts_dir) + write_script_by_frequency( + script_path, payload, self.frequency, self.scripts_dir + ) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 9bd85209c07..899b4e5e632 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -15,13 +15,15 @@ from cloudinit import log as logging from cloudinit import net, sources, type_utils, util from cloudinit.event import EventScope, EventType, userdata_to_events + # Default handlers (used if not overridden) from cloudinit.handlers.boot_hook import BootHookPartHandler from cloudinit.handlers.cloud_config import CloudConfigPartHandler from cloudinit.handlers.jinja_template import JinjaTemplatePartHandler from cloudinit.handlers.shell_script import ShellScriptPartHandler -from cloudinit.handlers.shell_script_by_frequency import \ - ShellScriptByFreqPartHandler +from cloudinit.handlers.shell_script_by_frequency import ( + ShellScriptByFreqPartHandler, +) from cloudinit.handlers.upstart_job import UpstartJobPartHandler from cloudinit.net import cmdline from cloudinit.reporting import events @@ -518,12 +520,15 @@ def _default_handlers(self, opts=None): # TODO(harlowja) Hmmm, should we dynamically import these?? cloudconfig_handler = CloudConfigPartHandler(**opts) shellscript_handler = ShellScriptPartHandler(**opts) - shellscript_per_boot_handler = \ - ShellScriptByFreqPartHandler(PER_ALWAYS, **opts) - shellscript_per_instance_handler = \ - ShellScriptByFreqPartHandler(PER_INSTANCE, **opts) - shellscript_per_once_handler = \ - ShellScriptByFreqPartHandler(PER_ONCE, **opts) + shellscript_per_boot_handler = ShellScriptByFreqPartHandler( + PER_ALWAYS, **opts + ) + shellscript_per_instance_handler = ShellScriptByFreqPartHandler( + PER_INSTANCE, **opts + ) + shellscript_per_once_handler = ShellScriptByFreqPartHandler( + PER_ONCE, **opts + ) def_handlers = [ cloudconfig_handler, shellscript_handler, diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 556437861eb..1e373308c0c 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -101,7 +101,7 @@ def disable_subp_usage(request): pass -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def session_cloud(): if integration_settings.PLATFORM not in platforms.keys(): raise ValueError( @@ -283,14 +283,14 @@ def client(request, fixture_utils, session_cloud, setup_image): yield client -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def module_client(request, fixture_utils, session_cloud, setup_image): """Provide a client that runs once per module.""" with _client(request, fixture_utils, session_cloud) as client: yield client -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def class_client(request, fixture_utils, session_cloud, setup_image): """Provide a client that runs once per class.""" with _client(request, fixture_utils, session_cloud) as client: diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index 6b20828f3a6..3e29d861933 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -1,22 +1,23 @@ - """Integration tests for various handlers.""" from io import StringIO -# from types import SimpleNamespace import pytest from cloudinit.cmd.devel.make_mime import create_mime_message from tests.integration_tests.instances import IntegrationInstance +# from types import SimpleNamespace + + PER_FREQ_TEMPLATE = """\ #!/bin/bash touch /tmp/test_per_freq_{} """ -PER_BOOT_FILE = StringIO(PER_FREQ_TEMPLATE.format('boot')) -PER_INSTANCE_FILE = StringIO(PER_FREQ_TEMPLATE.format('instance')) -PER_ONCE_FILE = StringIO(PER_FREQ_TEMPLATE.format('once')) +PER_BOOT_FILE = StringIO(PER_FREQ_TEMPLATE.format("boot")) +PER_INSTANCE_FILE = StringIO(PER_FREQ_TEMPLATE.format("instance")) +PER_ONCE_FILE = StringIO(PER_FREQ_TEMPLATE.format("once")) # args = SimpleNamespace( # debug=False, @@ -29,46 +30,46 @@ # ) FILES = [ - (PER_BOOT_FILE, 'boot.sh', 'x-shellscript-per-boot'), - (PER_INSTANCE_FILE, 'instance.sh', 'x-shellscript-per-instance'), - (PER_ONCE_FILE, 'once.sh', 'x-shellscript-per-once'), + (PER_BOOT_FILE, "boot.sh", "x-shellscript-per-boot"), + (PER_INSTANCE_FILE, "instance.sh", "x-shellscript-per-instance"), + (PER_ONCE_FILE, "once.sh", "x-shellscript-per-once"), ] USER_DATA, errors = create_mime_message(FILES) -print(f'errors={errors}') -print(f'USER_DATA=${USER_DATA}') +print(f"errors={errors}") +print(f"USER_DATA=${USER_DATA}") @pytest.mark.user_data(USER_DATA) def test_per_freq(client: IntegrationInstance): - print('checking /v/l/c/scripts exists ...') - rc_ok = client.execute('test -d /var/lib/cloud/scripts').ok + print("checking /v/l/c/scripts exists ...") + rc_ok = client.execute("test -d /var/lib/cloud/scripts").ok assert rc_ok is True # Test per-boot - print('checking /v/l/c/s/per-boot/boot.sh exists ...') - cmd = 'test -f /var/lib/cloud/scripts/per-boot/boot.sh' + print("checking /v/l/c/s/per-boot/boot.sh exists ...") + cmd = "test -f /var/lib/cloud/scripts/per-boot/boot.sh" rc_ok = client.execute(cmd).ok assert rc_ok is True - print('checking /tmp/c/test_per_freq_boot exists ...') - cmd = 'test -f /tmp/test_per_freq_boot' + print("checking /tmp/c/test_per_freq_boot exists ...") + cmd = "test -f /tmp/test_per_freq_boot" rc_ok = client.execute(cmd).ok assert rc_ok is True # Test per-instance - print('checking /v/l/c/s/per-boot/instance.sh exists ...') - cmd = 'test -f /var/lib/cloud/scripts/per-boot/instance.sh' + print("checking /v/l/c/s/per-boot/instance.sh exists ...") + cmd = "test -f /var/lib/cloud/scripts/per-boot/instance.sh" rc_ok = client.execute(cmd).ok assert rc_ok is True - print('checking /tmp/c/test_per_freq_instance exists ...') - cmd = 'test -f /tmp/test_per_freq_instance' + print("checking /tmp/c/test_per_freq_instance exists ...") + cmd = "test -f /tmp/test_per_freq_instance" rc_ok = client.execute(cmd).ok assert rc_ok is True # Test per-once - print('checking /v/l/c/s/per-boot/once.sh exists ...') - cmd = 'test -f /var/lib/cloud/scripts/per-boot/once.sh' + print("checking /v/l/c/s/per-boot/once.sh exists ...") + cmd = "test -f /var/lib/cloud/scripts/per-boot/once.sh" rc_ok = client.execute(cmd).ok assert rc_ok is True - print('checking /tmp/c/test_per_freq_once exists ...') - cmd = 'test -f /tmp/test_per_freq_once' + print("checking /tmp/c/test_per_freq_once exists ...") + cmd = "test -f /tmp/test_per_freq_once" rc_ok = client.execute(cmd).ok assert rc_ok is True pass diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 0b106fd8246..932399162da 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -20,8 +20,9 @@ ) from cloudinit.handlers.shell_script import ShellScriptPartHandler from cloudinit.handlers.upstart_job import UpstartJobPartHandler + # from cloudinit.settings import PER_ALWAYS, PER_INSTANCE -from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, PER_ONCE) +from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE from tests.unittests.helpers import ( CiTestCase, FilesystemMockingTestCase, @@ -479,11 +480,13 @@ class TestShellScriptByFrequencyHandlers(CiTestCase): def do_test_frequency(self, frequency): from cloudinit.cmd.devel import read_cfg_paths - from cloudinit.handlers.shell_script_by_frequency \ - import (get_script_folder_by_frequency, - pathMap) + from cloudinit.handlers.shell_script_by_frequency import ( + get_script_folder_by_frequency, + pathMap, + ) + ci_paths = read_cfg_paths() - scripts_dir = ci_paths.get_cpath('scripts') + scripts_dir = ci_paths.get_cpath("scripts") testFolder = os.path.join(scripts_dir, pathMap[frequency]) folder = get_script_folder_by_frequency(frequency, scripts_dir) self.assertEqual(testFolder, folder) From 854a87191306a34df16899f7478aecad7ba3ab8a Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Thu, 20 Jan 2022 19:29:38 -0600 Subject: [PATCH 33/49] Incorporated changes from James Falcons suggestions on 1/11/22 --- .gitignore | 1 - .../integration_tests/shellscript-by-freq.py | 44 ++++--------------- tests/unittests/test_builtin_handlers.py | 18 +++----- 3 files changed, 16 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index 046ccb2619c..6e8b7bf6663 100644 --- a/.gitignore +++ b/.gitignore @@ -33,5 +33,4 @@ cloud-init_*.tar.xz cloud-init_*.upload # user test settings -tests/integration_tests/user_settings.py misc/ diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index 3e29d861933..b2cb250f515 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -7,9 +7,6 @@ from cloudinit.cmd.devel.make_mime import create_mime_message from tests.integration_tests.instances import IntegrationInstance -# from types import SimpleNamespace - - PER_FREQ_TEMPLATE = """\ #!/bin/bash touch /tmp/test_per_freq_{} @@ -19,16 +16,6 @@ PER_INSTANCE_FILE = StringIO(PER_FREQ_TEMPLATE.format("instance")) PER_ONCE_FILE = StringIO(PER_FREQ_TEMPLATE.format("once")) -# args = SimpleNamespace( -# debug=False, -# list_types=False, -# files=[ -# (PER_BOOT_FILE, 'boot.sh', 'x-shellscript-per-boot'), -# (PER_INSTANCE_FILE, 'instance.sh', 'x-shellscript-per-instance'), -# (PER_ONCE_FILE, 'once.sh', 'x-shellscript-per-once'), -# ] -# ) - FILES = [ (PER_BOOT_FILE, "boot.sh", "x-shellscript-per-boot"), (PER_INSTANCE_FILE, "instance.sh", "x-shellscript-per-instance"), @@ -42,34 +29,21 @@ @pytest.mark.user_data(USER_DATA) def test_per_freq(client: IntegrationInstance): - print("checking /v/l/c/scripts exists ...") - rc_ok = client.execute("test -d /var/lib/cloud/scripts").ok - assert rc_ok is True + # Sanity test for scripts folder + cmd = "test -d /var/lib/cloud/scripts" + assert client.execute(cmd).ok # Test per-boot - print("checking /v/l/c/s/per-boot/boot.sh exists ...") cmd = "test -f /var/lib/cloud/scripts/per-boot/boot.sh" - rc_ok = client.execute(cmd).ok - assert rc_ok is True - print("checking /tmp/c/test_per_freq_boot exists ...") + assert client.execute(cmd).ok cmd = "test -f /tmp/test_per_freq_boot" - rc_ok = client.execute(cmd).ok - assert rc_ok is True + assert client.execute(cmd).ok # Test per-instance - print("checking /v/l/c/s/per-boot/instance.sh exists ...") cmd = "test -f /var/lib/cloud/scripts/per-boot/instance.sh" - rc_ok = client.execute(cmd).ok - assert rc_ok is True - print("checking /tmp/c/test_per_freq_instance exists ...") + assert client.execute(cmd).ok cmd = "test -f /tmp/test_per_freq_instance" - rc_ok = client.execute(cmd).ok - assert rc_ok is True + assert client.execute(cmd).ok # Test per-once - print("checking /v/l/c/s/per-boot/once.sh exists ...") cmd = "test -f /var/lib/cloud/scripts/per-boot/once.sh" - rc_ok = client.execute(cmd).ok - assert rc_ok is True - print("checking /tmp/c/test_per_freq_once exists ...") + assert client.execute(cmd).ok cmd = "test -f /tmp/test_per_freq_once" - rc_ok = client.execute(cmd).ok - assert rc_ok is True - pass + assert client.execute(cmd).ok diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 932399162da..07d91c8e513 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -12,6 +12,7 @@ import pytest from cloudinit import handlers, helpers, subp, util +from cloudinit.cmd.devel import read_cfg_paths from cloudinit.handlers.cloud_config import CloudConfigPartHandler from cloudinit.handlers.jinja_template import ( JinjaTemplatePartHandler, @@ -19,8 +20,11 @@ render_jinja_payload, ) from cloudinit.handlers.shell_script import ShellScriptPartHandler +from cloudinit.handlers.shell_script_by_frequency import ( + get_script_folder_by_frequency, + pathMap, +) from cloudinit.handlers.upstart_job import UpstartJobPartHandler - # from cloudinit.settings import PER_ALWAYS, PER_INSTANCE from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE from tests.unittests.helpers import ( @@ -98,9 +102,7 @@ def test_upstart_frequency_single(self): ) -class TestJinjaTemplatePartHandler(CiTestCase): - - with_logs = True +class TestJinjaTemplatePartHandler(): mpath = "cloudinit.handlers.jinja_template." @@ -479,17 +481,11 @@ class TestShellScriptByFrequencyHandlers(CiTestCase): with_logs = True def do_test_frequency(self, frequency): - from cloudinit.cmd.devel import read_cfg_paths - from cloudinit.handlers.shell_script_by_frequency import ( - get_script_folder_by_frequency, - pathMap, - ) - ci_paths = read_cfg_paths() scripts_dir = ci_paths.get_cpath("scripts") testFolder = os.path.join(scripts_dir, pathMap[frequency]) folder = get_script_folder_by_frequency(frequency, scripts_dir) - self.assertEqual(testFolder, folder) + assert testFolder == folder def test_get_script_folder_per_boot(self): self.do_test_frequency(PER_ALWAYS) From e5db3cd7b45786cde733eac42c55859411230610 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Thu, 20 Jan 2022 19:31:42 -0600 Subject: [PATCH 34/49] final changes to pass tox -e format --- tests/unittests/test_builtin_handlers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 07d91c8e513..1d305056a86 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -25,6 +25,7 @@ pathMap, ) from cloudinit.handlers.upstart_job import UpstartJobPartHandler + # from cloudinit.settings import PER_ALWAYS, PER_INSTANCE from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE from tests.unittests.helpers import ( @@ -102,7 +103,7 @@ def test_upstart_frequency_single(self): ) -class TestJinjaTemplatePartHandler(): +class TestJinjaTemplatePartHandler: mpath = "cloudinit.handlers.jinja_template." From 8aed6f2c13c57f5d7a11237c9eef4a6fa7a7f880 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Thu, 20 Jan 2022 21:12:25 -0600 Subject: [PATCH 35/49] Fixed mistake I made in changing test_builtin_handlers that was breaking Jinja tests --- tests/unittests/test_builtin_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 1d305056a86..b565541361a 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -103,7 +103,7 @@ def test_upstart_frequency_single(self): ) -class TestJinjaTemplatePartHandler: +class TestJinjaTemplatePartHandler(CiTestCase): mpath = "cloudinit.handlers.jinja_template." @@ -478,7 +478,7 @@ def test_render_jinja_payload_replaces_missing_variables_and_warns(self): self.assertIn(expected_log, self.logs.getvalue()) -class TestShellScriptByFrequencyHandlers(CiTestCase): +class TestShellScriptByFrequencyHandlers(): with_logs = True def do_test_frequency(self, frequency): From c5f677f49cc23703cb711ab9f69a076d66069d59 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Thu, 20 Jan 2022 21:49:46 -0600 Subject: [PATCH 36/49] Take 2: Fixed mistake I made in changing test_builtin_handlers that was breaking Jinja tests --- tests/unittests/test_builtin_handlers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index b565541361a..0f2da6694c7 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -105,6 +105,8 @@ def test_upstart_frequency_single(self): class TestJinjaTemplatePartHandler(CiTestCase): + with_logs = True + mpath = "cloudinit.handlers.jinja_template." def setUp(self): From 7077f3d56590066a6fe8d433ff70e514b3d01bd1 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Fri, 21 Jan 2022 04:09:38 +0000 Subject: [PATCH 37/49] flake8 + black --- tests/unittests/test_builtin_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 0f2da6694c7..b29238780c7 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -106,7 +106,7 @@ def test_upstart_frequency_single(self): class TestJinjaTemplatePartHandler(CiTestCase): with_logs = True - + mpath = "cloudinit.handlers.jinja_template." def setUp(self): @@ -480,7 +480,7 @@ def test_render_jinja_payload_replaces_missing_variables_and_warns(self): self.assertIn(expected_log, self.logs.getvalue()) -class TestShellScriptByFrequencyHandlers(): +class TestShellScriptByFrequencyHandlers: with_logs = True def do_test_frequency(self, frequency): From d072fcd11cdbdac0b161c3d68cf155725f568898 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Sun, 23 Jan 2022 13:23:26 -0600 Subject: [PATCH 38/49] one more commit; tbh this has changes I thought were committed --- .gitignore | 2 +- tests/integration_tests/shellscript-by-freq.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index d4ba3c200d4..7e38138b8b3 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,4 @@ cloud-init_*.tar.xz cloud-init_*.upload # user test settings -misc/ +tests/integration_tests/user_settings.py \ No newline at end of file diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index b2cb250f515..394c3a12e24 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -23,8 +23,6 @@ ] USER_DATA, errors = create_mime_message(FILES) -print(f"errors={errors}") -print(f"USER_DATA=${USER_DATA}") @pytest.mark.user_data(USER_DATA) From e3645ac6dc269fd8a8c84e4b4e7af1f9af04d385 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Tue, 25 Jan 2022 19:53:58 -0600 Subject: [PATCH 39/49] print() removal --- cloudinit/cmd/devel/make_mime.py | 4 ---- cloudinit/handlers/shell_script_by_frequency.py | 5 ----- 2 files changed, 9 deletions(-) diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index 3a0181fbe60..c7671a93385 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -21,12 +21,9 @@ def create_mime_message(files): - print(f"files={files}") - print(f"len(files)={len(files)}") sub_messages = [] errors = [] for i, (fh, filename, format_type) in enumerate(files): - print(f"files[{i}]={filename}") contents = fh.read() sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) sub_message.add_header( @@ -41,7 +38,6 @@ def create_mime_message(files): sub_messages.append(sub_message) combined_message = MIMEMultipart() for msg in sub_messages: - print("attaching message") combined_message.attach(msg) return (combined_message, errors) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index 71d8031d5c3..d68c06853e8 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -29,13 +29,11 @@ def get_script_folder_by_frequency(freq, scripts_dir): def write_script_by_frequency(script_path, payload, frequency, scripts_dir): """Given a filename, a payload, a frequency, and a scripts folder, write the payload to the correct frequency-specific path""" - print("write_script_by_frequency()") filename = os.path.basename(script_path) filename = util.clean_filename(filename) folder = get_script_folder_by_frequency(frequency, scripts_dir) path = os.path.join(folder, filename) payload = util.dos2unix(payload) - print(f"Writing payload to {path}") util.write_file(path, payload, 0o700) @@ -49,15 +47,12 @@ class ShellScriptByFreqPartHandler(Handler): ] def __init__(self, freq, paths, **_kwargs): - print("ShellScriptByFreqPartHandler: c'tor()") Handler.__init__(self, freq) self.scripts_dir = paths.get_cpath("scripts") if "script_path" in _kwargs: self.scripts_dir = paths.get_cpath(_kwargs["script_path"]) def handle_part(self, data, ctype, script_path, payload, frequency): - print("ShellScriptByFreqPartHandler.handle_part()") - print("script_path=%s", script_path) if script_path is not None: filename = os.path.basename(script_path) filename = util.clean_filename(filename) From 29442eca321f467005eff95a1772795a12942a4c Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Tue, 25 Jan 2022 21:10:03 -0600 Subject: [PATCH 40/49] falcojrs more significant suggestions --- .../handlers/shell_script_by_frequency.py | 37 ++++++++++++++----- cloudinit/stages.py | 19 +++------- .../integration_tests/shellscript-by-freq.py | 10 ++--- 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index d68c06853e8..fd2b4486617 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -40,14 +40,9 @@ def write_script_by_frequency(script_path, payload, frequency, scripts_dir): class ShellScriptByFreqPartHandler(Handler): """Common base class for the frequency-specific script handlers.""" - prefixes = [ - "text/x-shellscript-per-boot", - "text/x-shellscript-per-instance", - "text/x-shellscript-per-once", - ] - - def __init__(self, freq, paths, **_kwargs): - Handler.__init__(self, freq) + def __init__(self, script_frequency, paths, **_kwargs): + self.freq = script_frequency + Handler.__init__(self, PER_ALWAYS) self.scripts_dir = paths.get_cpath("scripts") if "script_path" in _kwargs: self.scripts_dir = paths.get_cpath(_kwargs["script_path"]) @@ -57,5 +52,29 @@ def handle_part(self, data, ctype, script_path, payload, frequency): filename = os.path.basename(script_path) filename = util.clean_filename(filename) write_script_by_frequency( - script_path, payload, self.frequency, self.scripts_dir + script_path, payload, self.script_frequency, self.scripts_dir ) + + +class ShellScriptPerBootPartHandler(ShellScriptByFreqPartHandler): + prefixes = ["text/x-shellscript-per-boot"] + + def __init__(self, paths, **kwargs): + ShellScriptByFreqPartHandler.__init__( + self, PER_ALWAYS, paths, **kwargs) + + +class ShellScriptPerInstancePartHandler(ShellScriptByFreqPartHandler): + prefixes = ["text/x-shellscript-per-instance"] + + def __init__(self, paths, **kwargs): + ShellScriptByFreqPartHandler.__init__( + self, PER_INSTANCE, paths, **kwargs) + + +class ShellScriptPerOncePartHandler(ShellScriptByFreqPartHandler): + prefixes = ["text/x-shellscript-per-once"] + + def __init__(self, paths, **kwargs): + ShellScriptByFreqPartHandler.__init__( + self, PER_ONCE, paths, **kwargs) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 899b4e5e632..6288b654394 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -22,7 +22,9 @@ from cloudinit.handlers.jinja_template import JinjaTemplatePartHandler from cloudinit.handlers.shell_script import ShellScriptPartHandler from cloudinit.handlers.shell_script_by_frequency import ( - ShellScriptByFreqPartHandler, + ShellScriptPerBootPartHandler, + ShellScriptPerInstancePartHandler, + ShellScriptPerOncePartHandler, ) from cloudinit.handlers.upstart_job import UpstartJobPartHandler from cloudinit.net import cmdline @@ -520,21 +522,12 @@ def _default_handlers(self, opts=None): # TODO(harlowja) Hmmm, should we dynamically import these?? cloudconfig_handler = CloudConfigPartHandler(**opts) shellscript_handler = ShellScriptPartHandler(**opts) - shellscript_per_boot_handler = ShellScriptByFreqPartHandler( - PER_ALWAYS, **opts - ) - shellscript_per_instance_handler = ShellScriptByFreqPartHandler( - PER_INSTANCE, **opts - ) - shellscript_per_once_handler = ShellScriptByFreqPartHandler( - PER_ONCE, **opts - ) def_handlers = [ cloudconfig_handler, shellscript_handler, - shellscript_per_boot_handler, - shellscript_per_instance_handler, - shellscript_per_once_handler, + ShellScriptPerBootPartHandler(**opts), + ShellScriptPerInstancePartHandler(**opts), + ShellScriptPerOncePartHandler(**opts), BootHookPartHandler(**opts), UpstartJobPartHandler(**opts), ] diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index 394c3a12e24..eef301c001a 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -12,12 +12,12 @@ touch /tmp/test_per_freq_{} """ -PER_BOOT_FILE = StringIO(PER_FREQ_TEMPLATE.format("boot")) +PER_ALWAYS_FILE = StringIO(PER_FREQ_TEMPLATE.format("always")) PER_INSTANCE_FILE = StringIO(PER_FREQ_TEMPLATE.format("instance")) PER_ONCE_FILE = StringIO(PER_FREQ_TEMPLATE.format("once")) FILES = [ - (PER_BOOT_FILE, "boot.sh", "x-shellscript-per-boot"), + (PER_ALWAYS_FILE, "always.sh", "x-shellscript-per-boot"), (PER_INSTANCE_FILE, "instance.sh", "x-shellscript-per-instance"), (PER_ONCE_FILE, "once.sh", "x-shellscript-per-once"), ] @@ -31,17 +31,17 @@ def test_per_freq(client: IntegrationInstance): cmd = "test -d /var/lib/cloud/scripts" assert client.execute(cmd).ok # Test per-boot - cmd = "test -f /var/lib/cloud/scripts/per-boot/boot.sh" + cmd = "test -f /var/lib/cloud/scripts/per-boot/always.sh" assert client.execute(cmd).ok cmd = "test -f /tmp/test_per_freq_boot" assert client.execute(cmd).ok # Test per-instance - cmd = "test -f /var/lib/cloud/scripts/per-boot/instance.sh" + cmd = "test -f /var/lib/cloud/scripts/per-instance/instance.sh" assert client.execute(cmd).ok cmd = "test -f /tmp/test_per_freq_instance" assert client.execute(cmd).ok # Test per-once - cmd = "test -f /var/lib/cloud/scripts/per-boot/once.sh" + cmd = "test -f /var/lib/cloud/scripts/per-once/once.sh" assert client.execute(cmd).ok cmd = "test -f /tmp/test_per_freq_once" assert client.execute(cmd).ok From e1cf5690fd4707635fd27cd3f34bcae8c8e42e7d Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Tue, 25 Jan 2022 21:37:23 -0600 Subject: [PATCH 41/49] fixed flake8 pylint and black tests --- cloudinit/handlers/shell_script_by_frequency.py | 11 ++++++----- cloudinit/stages.py | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index fd2b4486617..2e260f96551 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -41,7 +41,7 @@ class ShellScriptByFreqPartHandler(Handler): """Common base class for the frequency-specific script handlers.""" def __init__(self, script_frequency, paths, **_kwargs): - self.freq = script_frequency + self.script_frequency = script_frequency Handler.__init__(self, PER_ALWAYS) self.scripts_dir = paths.get_cpath("scripts") if "script_path" in _kwargs: @@ -61,7 +61,8 @@ class ShellScriptPerBootPartHandler(ShellScriptByFreqPartHandler): def __init__(self, paths, **kwargs): ShellScriptByFreqPartHandler.__init__( - self, PER_ALWAYS, paths, **kwargs) + self, PER_ALWAYS, paths, **kwargs + ) class ShellScriptPerInstancePartHandler(ShellScriptByFreqPartHandler): @@ -69,12 +70,12 @@ class ShellScriptPerInstancePartHandler(ShellScriptByFreqPartHandler): def __init__(self, paths, **kwargs): ShellScriptByFreqPartHandler.__init__( - self, PER_INSTANCE, paths, **kwargs) + self, PER_INSTANCE, paths, **kwargs + ) class ShellScriptPerOncePartHandler(ShellScriptByFreqPartHandler): prefixes = ["text/x-shellscript-per-once"] def __init__(self, paths, **kwargs): - ShellScriptByFreqPartHandler.__init__( - self, PER_ONCE, paths, **kwargs) + ShellScriptByFreqPartHandler.__init__(self, PER_ONCE, paths, **kwargs) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 6288b654394..ed39b8fb458 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -32,7 +32,6 @@ from cloudinit.settings import ( CLOUD_CONFIG, FREQUENCIES, - PER_ALWAYS, PER_INSTANCE, PER_ONCE, RUN_CLOUD_CONFIG, From c0fb855a8a2c464d5f2421443ed7662e36780e59 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Tue, 25 Jan 2022 21:39:19 -0600 Subject: [PATCH 42/49] slight change to test --- tests/integration_tests/shellscript-by-freq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index eef301c001a..f699382341d 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -33,7 +33,7 @@ def test_per_freq(client: IntegrationInstance): # Test per-boot cmd = "test -f /var/lib/cloud/scripts/per-boot/always.sh" assert client.execute(cmd).ok - cmd = "test -f /tmp/test_per_freq_boot" + cmd = "test -f /tmp/test_per_freq_always" assert client.execute(cmd).ok # Test per-instance cmd = "test -f /var/lib/cloud/scripts/per-instance/instance.sh" From 4a6b25e0734840586cfa0d74b487db61f5c34990 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Tue, 25 Jan 2022 22:22:12 -0600 Subject: [PATCH 43/49] added trailing newline to .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7e38138b8b3..39172d9a7e7 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,5 @@ cloud-init_*.tar.xz cloud-init_*.upload # user test settings -tests/integration_tests/user_settings.py \ No newline at end of file +tests/integration_tests/user_settings.py + From 0e8f81b6f4a8312286444b48a300c3c191a548ca Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Wed, 26 Jan 2022 00:09:04 -0600 Subject: [PATCH 44/49] marking my int tests with @pytest.mark.ci --- tests/integration_tests/shellscript-by-freq.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/shellscript-by-freq.py index f699382341d..25157722881 100644 --- a/tests/integration_tests/shellscript-by-freq.py +++ b/tests/integration_tests/shellscript-by-freq.py @@ -25,6 +25,7 @@ USER_DATA, errors = create_mime_message(FILES) +@pytest.mark.ci @pytest.mark.user_data(USER_DATA) def test_per_freq(client: IntegrationInstance): # Sanity test for scripts folder From bd3b7b8ddc36040ca0cc6677818f37430fc2e2f1 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Wed, 26 Jan 2022 01:17:17 -0600 Subject: [PATCH 45/49] renamed test file --- .../{shellscript-by-freq.py => test_shell_script_by_frequency.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/integration_tests/{shellscript-by-freq.py => test_shell_script_by_frequency.py} (100%) diff --git a/tests/integration_tests/shellscript-by-freq.py b/tests/integration_tests/test_shell_script_by_frequency.py similarity index 100% rename from tests/integration_tests/shellscript-by-freq.py rename to tests/integration_tests/test_shell_script_by_frequency.py From 383731f7d5d158e33171c71dce223bf47d51d236 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Wed, 26 Jan 2022 10:40:27 -0600 Subject: [PATCH 46/49] changes for falconjrs comments --- .gitignore | 1 - tests/unittests/test_builtin_handlers.py | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 39172d9a7e7..6eae45c98e9 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,3 @@ cloud-init_*.upload # user test settings tests/integration_tests/user_settings.py - diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index b29238780c7..987efe3815b 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -5,12 +5,10 @@ import copy import errno import os +import pytest import shutil import tempfile from textwrap import dedent - -import pytest - from cloudinit import handlers, helpers, subp, util from cloudinit.cmd.devel import read_cfg_paths from cloudinit.handlers.cloud_config import CloudConfigPartHandler @@ -25,8 +23,6 @@ pathMap, ) from cloudinit.handlers.upstart_job import UpstartJobPartHandler - -# from cloudinit.settings import PER_ALWAYS, PER_INSTANCE from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE from tests.unittests.helpers import ( CiTestCase, @@ -37,7 +33,6 @@ INSTANCE_DATA_FILE = "instance-data-sensitive.json" - class TestUpstartJobPartHandler(FilesystemMockingTestCase): mpath = "cloudinit.handlers.upstart_job." @@ -481,7 +476,6 @@ def test_render_jinja_payload_replaces_missing_variables_and_warns(self): class TestShellScriptByFrequencyHandlers: - with_logs = True def do_test_frequency(self, frequency): ci_paths = read_cfg_paths() From 7ea8dd4ea5081fbf8be47beea9d7567ed5fa7dbd Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Wed, 26 Jan 2022 11:11:05 -0600 Subject: [PATCH 47/49] Fixed flake8 black & isort failures --- tests/unittests/test_builtin_handlers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 987efe3815b..f1b42f592a9 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -5,10 +5,12 @@ import copy import errno import os -import pytest import shutil import tempfile from textwrap import dedent + +import pytest + from cloudinit import handlers, helpers, subp, util from cloudinit.cmd.devel import read_cfg_paths from cloudinit.handlers.cloud_config import CloudConfigPartHandler @@ -33,6 +35,7 @@ INSTANCE_DATA_FILE = "instance-data-sensitive.json" + class TestUpstartJobPartHandler(FilesystemMockingTestCase): mpath = "cloudinit.handlers.upstart_job." @@ -476,7 +479,6 @@ def test_render_jinja_payload_replaces_missing_variables_and_warns(self): class TestShellScriptByFrequencyHandlers: - def do_test_frequency(self, frequency): ci_paths = read_cfg_paths() scripts_dir = ci_paths.get_cpath("scripts") From eae159dc7bd88e648d7108925b271fc3d7dfb299 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Wed, 26 Jan 2022 14:23:40 -0600 Subject: [PATCH 48/49] Consolidate 3 handler classes into 1 as per falconjr --- .../handlers/shell_script_by_frequency.py | 37 +++++-------------- cloudinit/stages.py | 11 +++--- 2 files changed, 14 insertions(+), 34 deletions(-) diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py index 2e260f96551..923cca57d91 100644 --- a/cloudinit/handlers/shell_script_by_frequency.py +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -11,17 +11,22 @@ # specific folders in /v/l/c/scripts. It might make sense to expose this at a # higher level or in a more general module -- eg maybe in cloudinit/settings.py # itself -- but for now it's here. -pathMap = { +path_map = { PER_ALWAYS: "per-boot", PER_INSTANCE: "per-instance", PER_ONCE: "per-once", } +def get_mime_type_by_frequency(freq): + mime_type = f"text/x-shellscript-{path_map[freq]}" + return mime_type + + def get_script_folder_by_frequency(freq, scripts_dir): """Return the frequency-specific subfolder for a given frequency constant and parent folder.""" - freqPath = pathMap[freq] + freqPath = path_map[freq] folder = os.path.join(scripts_dir, freqPath) return folder @@ -41,8 +46,9 @@ class ShellScriptByFreqPartHandler(Handler): """Common base class for the frequency-specific script handlers.""" def __init__(self, script_frequency, paths, **_kwargs): - self.script_frequency = script_frequency Handler.__init__(self, PER_ALWAYS) + self.prefixes = [get_mime_type_by_frequency(script_frequency)] + self.script_frequency = script_frequency self.scripts_dir = paths.get_cpath("scripts") if "script_path" in _kwargs: self.scripts_dir = paths.get_cpath(_kwargs["script_path"]) @@ -54,28 +60,3 @@ def handle_part(self, data, ctype, script_path, payload, frequency): write_script_by_frequency( script_path, payload, self.script_frequency, self.scripts_dir ) - - -class ShellScriptPerBootPartHandler(ShellScriptByFreqPartHandler): - prefixes = ["text/x-shellscript-per-boot"] - - def __init__(self, paths, **kwargs): - ShellScriptByFreqPartHandler.__init__( - self, PER_ALWAYS, paths, **kwargs - ) - - -class ShellScriptPerInstancePartHandler(ShellScriptByFreqPartHandler): - prefixes = ["text/x-shellscript-per-instance"] - - def __init__(self, paths, **kwargs): - ShellScriptByFreqPartHandler.__init__( - self, PER_INSTANCE, paths, **kwargs - ) - - -class ShellScriptPerOncePartHandler(ShellScriptByFreqPartHandler): - prefixes = ["text/x-shellscript-per-once"] - - def __init__(self, paths, **kwargs): - ShellScriptByFreqPartHandler.__init__(self, PER_ONCE, paths, **kwargs) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ed39b8fb458..3f17294b944 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -22,9 +22,7 @@ from cloudinit.handlers.jinja_template import JinjaTemplatePartHandler from cloudinit.handlers.shell_script import ShellScriptPartHandler from cloudinit.handlers.shell_script_by_frequency import ( - ShellScriptPerBootPartHandler, - ShellScriptPerInstancePartHandler, - ShellScriptPerOncePartHandler, + ShellScriptByFreqPartHandler, ) from cloudinit.handlers.upstart_job import UpstartJobPartHandler from cloudinit.net import cmdline @@ -32,6 +30,7 @@ from cloudinit.settings import ( CLOUD_CONFIG, FREQUENCIES, + PER_ALWAYS, PER_INSTANCE, PER_ONCE, RUN_CLOUD_CONFIG, @@ -524,9 +523,9 @@ def _default_handlers(self, opts=None): def_handlers = [ cloudconfig_handler, shellscript_handler, - ShellScriptPerBootPartHandler(**opts), - ShellScriptPerInstancePartHandler(**opts), - ShellScriptPerOncePartHandler(**opts), + ShellScriptByFreqPartHandler(PER_ALWAYS, **opts), + ShellScriptByFreqPartHandler(PER_INSTANCE, **opts), + ShellScriptByFreqPartHandler(PER_ONCE, **opts), BootHookPartHandler(**opts), UpstartJobPartHandler(**opts), ] From e5f79d68efe727df2dccb27cd60d350007f91a2d Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Wed, 26 Jan 2022 14:32:01 -0600 Subject: [PATCH 49/49] fixed test failures (missed pathMap -> path_map) --- tests/unittests/test_builtin_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index f1b42f592a9..0dae924dbf7 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -22,7 +22,7 @@ from cloudinit.handlers.shell_script import ShellScriptPartHandler from cloudinit.handlers.shell_script_by_frequency import ( get_script_folder_by_frequency, - pathMap, + path_map, ) from cloudinit.handlers.upstart_job import UpstartJobPartHandler from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE @@ -482,7 +482,7 @@ class TestShellScriptByFrequencyHandlers: def do_test_frequency(self, frequency): ci_paths = read_cfg_paths() scripts_dir = ci_paths.get_cpath("scripts") - testFolder = os.path.join(scripts_dir, pathMap[frequency]) + testFolder = os.path.join(scripts_dir, path_map[frequency]) folder = get_script_folder_by_frequency(frequency, scripts_dir) assert testFolder == folder