first commit for first cloud-init PR

Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3-slim
+RUN apt-get update -y
+RUN apt-get install -y gcc git
+COPY ./requirements.txt /src/
+COPY ./test-requirements.txt /src/
+COPY ./integration-requirements.txt /src/
+WORKDIR /src
+RUN pip3 install --upgrade pip
+RUN pip3 install --requirement requirements.txt
+RUN pip3 install --requirement test-requirements.txt
+RUN pip3 install --requirement integration-requirements.txt
+RUN pip3 install flake8 pylint
+VOLUME /src
+ENTRYPOINT ["python", "-m"]
+

cloudinit-tester-build.sh

@@ -0,0 +1,7 @@
+#! /usr/bin/env sh
+
+# Use this script to build the docker image.
+# Not too fancy, but it does ensure the image gets a predictable name,
+# for use by other scripts.
+
+docker build --tag cloudinit-tester .

cloudinit-tester-cmds.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env sh
+
+# Use this script to run 3 useful invocations of cloudinit-tester-run.sh
+# It's easier to imagine a fancier script that asks Y/n before each test,
+# etc etc, but at that point it might make sense to switch to python
+
+# Flake8
+printf 'Running flake8 ...'
+./cloudinit-tester-run.sh flake8 --verbose cloudinit tests/unittests
+read -p 'flake8 done (press any key to continue ...)'
+
+# pylint
+printf 'Running pylint (this could take a while) ...'
+./cloudinit-tester-run.sh pylint cloudinit tests/unittests
+read -p '(press any key to continue ...)'
+
+# pytest
+printf 'Running pytest ...'
+./cloudinit-tester-run.sh pytest tests/unittests
+read -p '(press any key to continue ...)'

cloudinit-tester-run.sh

@@ -0,0 +1,8 @@
+# !/usr/bin/env sh
+
+# Use this script to run various commands inside a cloudinit-tester container.
+# The cloudinit-tester container has an ENTRYPOINT of 'python -m', so use the
+# script like this:
+#
+#    ./cloud-init-tester.sh pylint cloudinit  # This runs python -m pylint cloudinit
+docker run --rm --name cloudinit-tester --volume $PWD:/src cloudinit-tester "$@" 

cloudinit-tester-shell.sh

@@ -0,0 +1,10 @@
+#! /usr/bin/env sh
+
+# Use this script to startup a cloudinit-tester and then shell into it.
+# This is potentially useful if you want to run a large number of commands,
+# or play around.
+
+# It also supports cmdline args at the end, to allow for running arbitrary commands 
+# in case the default entrypoint of `python -m` is not suitable.
+
+docker run --rm --name cloudinit-tester --volume $PWD:/src -it --entrypoint /bin/bash cloudinit-tester "$@" 

cloudinit/cmd/devel/make_mime.py

@@ -17,6 +17,35 @@
           "-a script.sh:x-shellscript > user-data")
 
 
+def create_mime_message(args):
+    sub_messages = []
+    errors = []
+    rc = 0
+    for i, (fh, filename, format_type) in enumerate(args.files):
+        contents = fh.read()
+        sub_message = MIMEText(contents, format_type, sys.getdefaultencoding())
+        sub_message.add_header('Content-Disposition',
+                               'attachment; filename="%s"' % (filename))
+        content_type = sub_message.get_content_type().lower()
+        if content_type not in get_content_types():
+            level = "WARNING" if args.force else "ERROR"
+            msg = (level + ": content type %r for attachment %s "
+                   "may be incorrect!") % (content_type, i + 1)
+            sys.stderr.write(msg + '\n')
+            errors.append(msg)
+        sub_messages.append(sub_message)
+    if len(errors) and not args.force:
+        sys.stderr.write("Invalid content-types, override with --force\n")
+        combined_message = None
+        rc = 1
+    else:
+        combined_message = MIMEMultipart()
+        for msg in sub_messages:
+            combined_message.attach(msg)
+        rc = 0
+    return (combined_message, 1)
+
+
 def file_content_type(text):
     """ Return file content type by reading the first line of the input. """
     try:
@@ -62,7 +91,6 @@ def get_content_types(strip_prefix=False):
     return sorted([ctype.replace("text/", "") if strip_prefix else ctype
                    for ctype in INCLUSION_TYPES_MAP.values()])
 
-
 def handle_args(name, args):
     """Create a multi-part MIME archive for use as user-data.  Optionally
        print out the list of supported content types of cloud-init.
@@ -77,29 +105,9 @@ def handle_args(name, args):
         print("\n".join(get_content_types(strip_prefix=True)))
         return 0
 
-    sub_messages = []
-    errors = []
-    for i, (fh, filename, format_type) in enumerate(args.files):
-        contents = fh.read()
-        sub_message = MIMEText(contents, format_type, sys.getdefaultencoding())
-        sub_message.add_header('Content-Disposition',
-                               'attachment; filename="%s"' % (filename))
-        content_type = sub_message.get_content_type().lower()
-        if content_type not in get_content_types():
-            level = "WARNING" if args.force else "ERROR"
-            msg = (level + ": content type %r for attachment %s "
-                   "may be incorrect!") % (content_type, i + 1)
-            sys.stderr.write(msg + '\n')
-            errors.append(msg)
-        sub_messages.append(sub_message)
-    if len(errors) and not args.force:
-        sys.stderr.write("Invalid content-types, override with --force\n")
-        return 1
-    combined_message = MIMEMultipart()
-    for msg in sub_messages:
-        combined_message.attach(msg)
+    (combined_message, rc) = create_mime_message(args)
     print(combined_message)
-    return 0
+    return rc
 
 
 def main():

cloudinit/handlers/__init__.py

@@ -51,6 +51,13 @@
     '#cloud-config-archive': 'text/cloud-config-archive',
     '#cloud-config-jsonp': 'text/cloud-config-jsonp',
     '## template: jinja': 'text/jinja2',
+    # Note: for the next 3 entries, the prefix doesn't matter because these
+    # are for types that can only be used as part of a MIME message. However,
+    # including these entries supresses warnings during `cloudinit devel
+    # make-mime`, which otherwise would require `--force`.
+    'text/x-shellscript-per-boot': 'text/x-shellscript-per-boot',
+    'text/x-shellscript-per-instance': 'text/x-shellscript-per-instance',
+    'text/x-shellscript-per-once': 'text/x-shellscript-per-once'
 }
 
 # Sorted longest first

cloudinit/handlers/shell_script_by_frequency.py

@@ -0,0 +1,57 @@
+import os
+from cloudinit import log
+from cloudinit import util
+from cloudinit.handlers import Handler
+from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE
+LOG = log.getLogger(__name__)
+
+# cloutinit/settings.py defines PER_*** frequency constants. It makes sense to
+# use them here, instead hardcodes, and map them to the 'per-***' frequency-
+# specific folders in /v/l/c/scripts. It might make sense to expose this at a
+# higher level or in a more general module -- eg maybe in cloudinit/settings.py
+# itself -- but for now it's here.
+pathMap = {
+    PER_ALWAYS: 'per-boot',
+    PER_INSTANCE: 'per-instance',
+    PER_ONCE: 'per-once'
+}
+
+
+def get_script_folder_by_frequency(freq, scripts_dir):
+    """Return the frequency-specific subfolder for a given frequency constant
+       and parent folder."""
+    freqPath = pathMap[freq]
+    folder = os.path.join(scripts_dir, freqPath)
+    return folder
+
+
+def write_script_by_frequency(script_path, payload, frequency, scripts_dir):
+    """Given a filename, a payload, a frequency, and a scripts folder, write
+       the payload to the correct frequency-specific path"""
+    filename = os.path.basename(script_path)
+    filename = util.clean_filename(filename)
+    folder = get_script_folder_by_frequency(frequency, scripts_dir)
+    path = os.path.join(folder, filename)
+    payload = util.dos2unix(payload)
+    util.write_file(path, payload, 0o700)
+
+
+class ShellScriptByFreqPartHandler(Handler):
+    """Common base class for the frequency-specific script handlers."""
+    prefixes = ["text/x-shellscript-per-boot",
+                "text/x-shellscript-per-instance",
+                "text/x-shellscript-per-once"]
+
+    def __init__(self, freq, paths, **_kwargs):
+        Handler.__init__(self, freq)
+        self.scripts_dir = paths.get_cpath('scripts')
+        if 'script_path' in _kwargs:
+            self.scripts_dir = paths.get_cpath(_kwargs['script_path'])
+
+    def handle_part(self, data, ctype, script_path, payload, frequency):
+        if script_path is not None:
+            LOG.debug("script_path=%s", script_path)
+            filename = os.path.basename(script_path)
+            filename = util.clean_filename(filename)
+            write_script_by_frequency(script_path, payload, self.frequency,
+                                      self.scripts_dir)

cloudinit/stages.py

@@ -10,7 +10,8 @@
 import sys
 
 from cloudinit.settings import (
-    FREQUENCIES, CLOUD_CONFIG, PER_INSTANCE, RUN_CLOUD_CONFIG)
+    FREQUENCIES, CLOUD_CONFIG, PER_ALWAYS, PER_INSTANCE, PER_ONCE,
+    RUN_CLOUD_CONFIG)
 
 from cloudinit import handlers
 
@@ -19,6 +20,8 @@
 from cloudinit.handlers.cloud_config import CloudConfigPartHandler
 from cloudinit.handlers.jinja_template import JinjaTemplatePartHandler
 from cloudinit.handlers.shell_script import ShellScriptPartHandler
+from cloudinit.handlers.shell_script_by_frequency import \
+    ShellScriptByFreqPartHandler
 from cloudinit.handlers.upstart_job import UpstartJobPartHandler
 
 from cloudinit.event import EventType
@@ -415,9 +418,18 @@ def _default_handlers(self, opts=None):
         # TODO(harlowja) Hmmm, should we dynamically import these??
         cloudconfig_handler = CloudConfigPartHandler(**opts)
         shellscript_handler = ShellScriptPartHandler(**opts)
+        shellscript_per_boot_handler = \
+            ShellScriptByFreqPartHandler(PER_ALWAYS, **opts)
+        shellscript_per_instance_handler = \
+            ShellScriptByFreqPartHandler(PER_INSTANCE, **opts)
+        shellscript_per_once_handler = \
+            ShellScriptByFreqPartHandler(PER_ONCE, **opts)
         def_handlers = [
             cloudconfig_handler,
             shellscript_handler,
+            shellscript_per_boot_handler,
+            shellscript_per_instance_handler,
+            shellscript_per_once_handler,
             BootHookPartHandler(**opts),
             UpstartJobPartHandler(**opts),
         ]

tests/integration_tests/shellscript-by-freq.py

@@ -0,0 +1,36 @@
+
+"""Integration tests for various handlers."""
+
+from io import StringIO
+from types import SimpleNamespace
+
+import pytest
+
+from cloudinit.cmd.devel.make_mime import create_mime_message
+from tests.integration_tests.instances import IntegrationInstance
+
+PER_FREQ_TEMPLATE = """\
+#!/bin/bash
+touch /var/tmp/test_per_freq_{}
+"""
+
+PER_BOOT_FILE = StringIO(PER_FREQ_TEMPLATE.format('boot'))
+PER_INSTANCE_FILE = StringIO(PER_FREQ_TEMPLATE.format('instance'))
+PER_ONCE_FILE = StringIO(PER_FREQ_TEMPLATE.format('once'))
+
+args = SimpleNamespace(
+    debug=False,
+    list_types=False,
+    files=[
+        (PER_BOOT_FILE, 'boot.sh', 'x-shellscript-per-boot'),
+        (PER_INSTANCE_FILE, 'instance.sh', 'x-shellscript-per-instance'),
+        (PER_ONCE_FILE, 'once.sh', 'x-shellscript-per-once'),
+    ]
+)
+
+USER_DATA = create_mime_message(args)
+
+@pytest.mark.user_data(USER_DATA)
+def test_per_freq(client: IntegrationInstance):
+    scripts = client.execute('find /var/lib/cloud/scripts')
+    print(scripts)

tests/unittests/test_builtin_handlers.py

@@ -25,7 +25,7 @@
 from cloudinit.handlers.shell_script import ShellScriptPartHandler
 from cloudinit.handlers.upstart_job import UpstartJobPartHandler
 
-from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE)
+from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, PER_ONCE)
 
 
 class TestUpstartJobPartHandler(FilesystemMockingTestCase):
@@ -380,4 +380,29 @@ def test_render_jinja_payload_replaces_missing_variables_and_warns(self):
             " 'myfile': 'NOTHERE'")
         self.assertIn(expected_log, self.logs.getvalue())
 
+
+class TestShellScriptByFrequencyHandlers(CiTestCase):
+    with_logs = True
+
+    def do_test_frequency(self, frequency):
+        from cloudinit.cmd.devel import read_cfg_paths
+        from cloudinit.handlers.shell_script_by_frequency \
+            import (get_script_folder_by_frequency,
+                    pathMap)
+        ci_paths = read_cfg_paths()
+        scripts_dir = ci_paths.get_cpath('scripts')
+        testFolder = os.path.join(scripts_dir, pathMap[frequency])
+        folder = get_script_folder_by_frequency(frequency, scripts_dir)
+        self.assertEqual(testFolder, folder)
+
+    def test_get_script_folder_per_boot(self):
+        self.do_test_frequency(PER_ALWAYS)
+
+    def test_get_script_folder_per_instance(self):
+        self.do_test_frequency(PER_INSTANCE)
+
+    def test_get_script_folder_per_once(self):
+        self.do_test_frequency(PER_ONCE)
+
+
 # vi: ts=4 expandtab

tools/.github-cla-signers

@@ -5,6 +5,7 @@ Aman306
 andrewbogott
 antonyc
 aswinrajamannar
+beantaxi
 beezly
 bipinbachhao
 BirknerAlex